You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2015/01/06 13:35:37 UTC
svn commit: r1649794 - in
/manifoldcf/trunk/connectors/forcedmetadata/connector/src:
main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/
test/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/
Author: kwright
Date: Tue Jan 6 12:35:37 2015
New Revision: 1649794
URL: http://svn.apache.org/r1649794
Log:
Refactor for clarity
Added:
manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/FieldDataFactory.java (with props)
manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/FieldSource.java (with props)
manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/IDataSource.java (with props)
manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/StringSource.java (with props)
Modified:
manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ForcedMetadataConnector.java
manifoldcf/trunk/connectors/forcedmetadata/connector/src/test/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ExpressionTest.java
Added: manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/FieldDataFactory.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/FieldDataFactory.java?rev=1649794&view=auto
==============================================================================
--- manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/FieldDataFactory.java (added)
+++ manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/FieldDataFactory.java Tue Jan 6 12:35:37 2015
@@ -0,0 +1,133 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.agents.transformation.forcedmetadata;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+
+import java.io.*;
+import java.util.*;
+
+/** This class provides unique Reader and other field instances, when requested, based
+* on an input RepositoryDocument. It does this by pulling the values of the field into
+* a CharacterInput implementation, thus making a temporary file copy. So it is imperative
+* that this object is closed when it is no longer needed.
+*/
+public class FieldDataFactory {
+
+ protected final RepositoryDocument sourceDocument;
+
+ // Readers (organized by metadata)
+ protected final Map<String,CharacterInput[]> metadataReaders = new HashMap<String,CharacterInput[]>();
+
+ public FieldDataFactory(RepositoryDocument sourceDocument) {
+ this.sourceDocument = sourceDocument;
+ }
+
+ public void close()
+ throws ManifoldCFException {
+ for (String key : metadataReaders.keySet()) {
+ CharacterInput[] rt = metadataReaders.get(key);
+ for (CharacterInput r : rt) {
+ r.discard();
+ }
+ }
+ }
+
+ public Object[] getField(String fieldName)
+ throws IOException, ManifoldCFException {
+ CharacterInput[] inputs = metadataReaders.get(fieldName);
+ if (inputs == null) {
+ // Either never seen the field before, or it's not a Reader
+ Object[] fieldValues = sourceDocument.getField(fieldName);
+ if (fieldValues == null)
+ return fieldValues;
+ if (fieldValues instanceof Reader[]) {
+ // Create a copy
+ CharacterInput[] newValues = new CharacterInput[fieldValues.length];
+ try {
+ // Populate newValues
+ for (int i = 0; i < newValues.length; i++) {
+ newValues[i] = new TempFileCharacterInput((Reader)fieldValues[i]);
+ }
+ metadataReaders.put(fieldName,newValues);
+ inputs = newValues;
+ } catch (Throwable e) {
+ for (CharacterInput r : newValues)
+ {
+ if (r != null)
+ r.discard();
+ }
+ if (e instanceof IOException)
+ throw (IOException)e;
+ else if (e instanceof RuntimeException)
+ throw (RuntimeException)e;
+ else if (e instanceof Error)
+ throw (Error)e;
+ else
+ throw new RuntimeException("Unknown exception type: "+e.getClass().getName()+": "+e.getMessage(),e);
+ }
+ } else {
+ return fieldValues;
+ }
+ }
+
+ Reader[] newReaders = new Reader[inputs.length];
+ for (int i = 0; i < inputs.length; i++)
+ {
+ inputs[i].doneWithStream();
+ newReaders[i] = inputs[i].getStream();
+ }
+ return newReaders;
+ }
+
+ public String[] getFieldAsStrings(String fieldName)
+ throws IOException, ManifoldCFException {
+ CharacterInput[] cilist = metadataReaders.get(fieldName);
+ if (cilist == null)
+ return sourceDocument.getFieldAsStrings(fieldName);
+
+ // We've created a local array of CharacterInputs from this field. We'll need to convert these
+ // to strings.
+ char[] buffer = new char[65536];
+ String[] rval = new String[cilist.length];
+ for (int i = 0; i < rval.length; i++) {
+ CharacterInput ci = cilist[i];
+ ci.doneWithStream();
+ Reader r = ci.getStream();
+ // Read into a buffer
+ StringBuilder newValue = new StringBuilder();
+ while (true)
+ {
+ int amt = r.read(buffer);
+ if (amt == -1)
+ break;
+ newValue.append(buffer,0,amt);
+ }
+ rval[i] = newValue.toString();
+ }
+ sourceDocument.addField(fieldName,rval);
+ metadataReaders.remove(fieldName);
+ for (CharacterInput ci : cilist) {
+ ci.discard();
+ }
+ return rval;
+ }
+}
+
Propchange: manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/FieldDataFactory.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/FieldDataFactory.java
------------------------------------------------------------------------------
svn:keywords = Id
Added: manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/FieldSource.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/FieldSource.java?rev=1649794&view=auto
==============================================================================
--- manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/FieldSource.java (added)
+++ manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/FieldSource.java Tue Jan 6 12:35:37 2015
@@ -0,0 +1,57 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.agents.transformation.forcedmetadata;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+
+import java.io.*;
+import java.util.*;
+
+public class FieldSource implements IDataSource {
+
+ protected final FieldDataFactory rd;
+ protected final String fieldName;
+
+ public FieldSource(FieldDataFactory rd, String fieldName) {
+ this.rd = rd;
+ this.fieldName = fieldName;
+ }
+
+ @Override
+ public int getSize()
+ throws IOException, ManifoldCFException {
+ Object[] rawForm = getRawForm();
+ if (rawForm == null)
+ return 0;
+ return rawForm.length;
+ }
+
+ @Override
+ public Object[] getRawForm()
+ throws IOException, ManifoldCFException {
+ return rd.getField(fieldName);
+ }
+
+ @Override
+ public String[] getStringForm()
+ throws IOException, ManifoldCFException {
+ return rd.getFieldAsStrings(fieldName);
+ }
+}
Propchange: manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/FieldSource.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/FieldSource.java
------------------------------------------------------------------------------
svn:keywords = Id
Modified: manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ForcedMetadataConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ForcedMetadataConnector.java?rev=1649794&r1=1649793&r2=1649794&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ForcedMetadataConnector.java (original)
+++ manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ForcedMetadataConnector.java Tue Jan 6 12:35:37 2015
@@ -629,70 +629,6 @@ public class ForcedMetadataConnector ext
return input;
}
- public interface IDataSource {
- public int getSize() throws IOException, ManifoldCFException;
- public Object[] getRawForm() throws IOException, ManifoldCFException;
- public String[] getStringForm() throws IOException, ManifoldCFException;
- }
-
- protected static class StringSource implements IDataSource {
- protected final String[] data;
-
- public StringSource(String data) {
- this.data = new String[]{data};
- }
-
- public StringSource(String[] data) {
- this.data = data;
- }
-
- @Override
- public int getSize() {
- return data.length;
- }
-
- @Override
- public Object[] getRawForm() {
- return data;
- }
-
- @Override
- public String[] getStringForm() {
- return data;
- }
- }
-
- protected static class FieldSource implements IDataSource {
-
- protected final FieldDataFactory rd;
- protected final String fieldName;
-
- public FieldSource(FieldDataFactory rd, String fieldName) {
- this.rd = rd;
- this.fieldName = fieldName;
- }
-
- @Override
- public int getSize()
- throws IOException, ManifoldCFException {
- Object[] rawForm = getRawForm();
- if (rawForm == null)
- return 0;
- return rawForm.length;
- }
-
- @Override
- public Object[] getRawForm()
- throws IOException, ManifoldCFException {
- return rd.getField(fieldName);
- }
-
- @Override
- public String[] getStringForm()
- throws IOException, ManifoldCFException {
- return rd.getFieldAsStrings(fieldName);
- }
- }
protected static IDataSource append(IDataSource currentValues, IDataSource data)
throws IOException, ManifoldCFException {
@@ -874,117 +810,6 @@ public class ForcedMetadataConnector ext
}
}
- /** This class provides unique Reader and other field instances, when requested, based
- * on an input RepositoryDocument. It does this by pulling the values of the field into
- * a CharacterInput implementation, thus making a temporary file copy. So it is imperative
- * that this object is closed when it is no longer needed.
- */
- public static class FieldDataFactory {
-
- protected final RepositoryDocument sourceDocument;
-
- // Readers (organized by metadata)
- protected final Map<String,CharacterInput[]> metadataReaders = new HashMap<String,CharacterInput[]>();
-
- public FieldDataFactory(RepositoryDocument sourceDocument) {
- this.sourceDocument = sourceDocument;
- }
-
- public void close()
- throws ManifoldCFException
- {
- for (String key : metadataReaders.keySet())
- {
- CharacterInput[] rt = metadataReaders.get(key);
- for (CharacterInput r : rt)
- {
- r.discard();
- }
- }
- }
-
- public Object[] getField(String fieldName)
- throws IOException, ManifoldCFException {
- CharacterInput[] inputs = metadataReaders.get(fieldName);
- if (inputs == null) {
- // Either never seen the field before, or it's not a Reader
- Object[] fieldValues = sourceDocument.getField(fieldName);
- if (fieldValues == null)
- return fieldValues;
- if (fieldValues instanceof Reader[]) {
- // Create a copy
- CharacterInput[] newValues = new CharacterInput[fieldValues.length];
- try {
- // Populate newValues
- for (int i = 0; i < newValues.length; i++)
- {
- newValues[i] = new TempFileCharacterInput((Reader)fieldValues[i]);
- }
- metadataReaders.put(fieldName,newValues);
- inputs = newValues;
- } catch (Throwable e) {
- for (CharacterInput r : newValues)
- {
- if (r != null)
- r.discard();
- }
- if (e instanceof IOException)
- throw (IOException)e;
- else if (e instanceof RuntimeException)
- throw (RuntimeException)e;
- else if (e instanceof Error)
- throw (Error)e;
- else
- throw new RuntimeException("Unknown exception type: "+e.getClass().getName()+": "+e.getMessage(),e);
- }
- } else {
- return fieldValues;
- }
- }
-
- Reader[] newReaders = new Reader[inputs.length];
- for (int i = 0; i < inputs.length; i++)
- {
- inputs[i].doneWithStream();
- newReaders[i] = inputs[i].getStream();
- }
- return newReaders;
- }
-
- public String[] getFieldAsStrings(String fieldName)
- throws IOException, ManifoldCFException {
- CharacterInput[] cilist = metadataReaders.get(fieldName);
- if (cilist == null)
- return sourceDocument.getFieldAsStrings(fieldName);
-
- // We've created a local array of CharacterInputs from this field. We'll need to convert these
- // to strings.
- char[] buffer = new char[65536];
- String[] rval = new String[cilist.length];
- for (int i = 0; i < rval.length; i++) {
- CharacterInput ci = cilist[i];
- ci.doneWithStream();
- Reader r = ci.getStream();
- // Read into a buffer
- StringBuilder newValue = new StringBuilder();
- while (true)
- {
- int amt = r.read(buffer);
- if (amt == -1)
- break;
- newValue.append(buffer,0,amt);
- }
- rval[i] = newValue.toString();
- }
- sourceDocument.addField(fieldName,rval);
- metadataReaders.remove(fieldName);
- for (CharacterInput ci : cilist) {
- ci.discard();
- }
- return rval;
- }
- }
-
}
Added: manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/IDataSource.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/IDataSource.java?rev=1649794&view=auto
==============================================================================
--- manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/IDataSource.java (added)
+++ manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/IDataSource.java Tue Jan 6 12:35:37 2015
@@ -0,0 +1,33 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.agents.transformation.forcedmetadata;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+
+import java.io.*;
+import java.util.*;
+
+/** This interface describes a general field data source.
+*/
+public interface IDataSource {
+ public int getSize() throws IOException, ManifoldCFException;
+ public Object[] getRawForm() throws IOException, ManifoldCFException;
+ public String[] getStringForm() throws IOException, ManifoldCFException;
+}
Propchange: manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/IDataSource.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/IDataSource.java
------------------------------------------------------------------------------
svn:keywords = Id
Added: manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/StringSource.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/StringSource.java?rev=1649794&view=auto
==============================================================================
--- manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/StringSource.java (added)
+++ manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/StringSource.java Tue Jan 6 12:35:37 2015
@@ -0,0 +1,52 @@
+/* $Id$ */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.manifoldcf.agents.transformation.forcedmetadata;
+
+import org.apache.manifoldcf.core.interfaces.*;
+import org.apache.manifoldcf.agents.interfaces.*;
+
+import java.io.*;
+import java.util.*;
+
+public class StringSource implements IDataSource {
+ protected final String[] data;
+
+ public StringSource(String data) {
+ this.data = new String[]{data};
+ }
+
+ public StringSource(String[] data) {
+ this.data = data;
+ }
+
+ @Override
+ public int getSize() {
+ return data.length;
+ }
+
+ @Override
+ public Object[] getRawForm() {
+ return data;
+ }
+
+ @Override
+ public String[] getStringForm() {
+ return data;
+ }
+}
Propchange: manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/StringSource.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: manifoldcf/trunk/connectors/forcedmetadata/connector/src/main/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/StringSource.java
------------------------------------------------------------------------------
svn:keywords = Id
Modified: manifoldcf/trunk/connectors/forcedmetadata/connector/src/test/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ExpressionTest.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/forcedmetadata/connector/src/test/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ExpressionTest.java?rev=1649794&r1=1649793&r2=1649794&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/forcedmetadata/connector/src/test/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ExpressionTest.java (original)
+++ manifoldcf/trunk/connectors/forcedmetadata/connector/src/test/java/org/apache/manifoldcf/agents/transformation/forcedmetadata/ExpressionTest.java Tue Jan 6 12:35:37 2015
@@ -36,7 +36,7 @@ public class ExpressionTest
inputDoc.addField("stringfield",new String[]{"stringa","stringb","stringc"});
inputDoc.addField("readerfield",new Reader[]{new StringReader("readera"),new StringReader("readerb")});
inputDoc.addField("datefield",new Date[]{new Date(0L), new Date(100000000L)});
- ForcedMetadataConnector.FieldDataFactory fdf = new ForcedMetadataConnector.FieldDataFactory(inputDoc);
+ FieldDataFactory fdf = new FieldDataFactory(inputDoc);
try {
arrayEquals(new String[]{"stringa","stringb","stringc"}, (String[])(ForcedMetadataConnector.processExpression("${stringfield}", fdf).getRawForm()));
arrayEquals(new String[]{"prefixstringapostfix","prefixstringbpostfix","prefixstringcpostfix"}, (String[])(ForcedMetadataConnector.processExpression("prefix${stringfield}postfix", fdf).getRawForm()));