You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by sh...@apache.org on 2008/12/12 08:02:10 UTC
svn commit: r725926 - in /lucene/solr/trunk/contrib/dataimporthandler:
CHANGES.txt
src/main/java/org/apache/solr/handler/dataimport/ClobTransformer.java
src/main/java/org/apache/solr/handler/dataimport/FieldReaderDataSource.java
Author: shalin
Date: Thu Dec 11 23:02:09 2008
New Revision: 725926
URL: http://svn.apache.org/viewvc?rev=725926&view=rev
Log:
SOLR-891 -- A Transformer to read strings from Clob type
Added:
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ClobTransformer.java (with props)
Modified:
lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FieldReaderDataSource.java
Modified: lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt?rev=725926&r1=725925&r2=725926&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt Thu Dec 11 23:02:09 2008
@@ -38,6 +38,9 @@
6. SOLR-886: DataImportHandler should rollback when an import fails or it is aborted
(shalin)
+7. SOLR-891: A Transformer to read strings from Clob type.
+ (Noble Paul via shalin)
+
Optimizations
----------------------
1. SOLR-846: Reduce memory consumption during delta import by removing keys when used
Added: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ClobTransformer.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ClobTransformer.java?rev=725926&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ClobTransformer.java (added)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ClobTransformer.java Thu Dec 11 23:02:09 2008
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.sql.Clob;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Transformer instance which converts a Clob to a String.
+ * <p/>
+ * Refer to <a href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * <p/>
+ * <b>This API is experimental and subject to change</b>
+ *
+ * @version $Id$
+ * @since solr 1.4
+ */
+public class ClobTransformer extends Transformer {
+ public Object transformRow(Map<String, Object> aRow, Context context) {
+ for (Map<String, String> map : context.getAllEntityFields()) {
+ if (!"true".equals(map.get(CLOB))) continue;
+ String column = map.get(DataImporter.COLUMN);
+ String srcCol = map.get(RegexTransformer.SRC_COL_NAME);
+ if (srcCol == null)
+ srcCol = column;
+ Object o = aRow.get(srcCol);
+ if (o instanceof List) {
+ List<Clob> inputs = (List<Clob>) o;
+ List<String> results = new ArrayList<String>();
+ for (Object input : inputs) {
+ if (input instanceof Clob) {
+ Clob clob = (Clob) input;
+ results.add(readFromClob(clob));
+ }
+ }
+ aRow.put(column, results);
+ } else {
+ if (o instanceof Clob) {
+ Clob clob = (Clob) o;
+ aRow.put(column, readFromClob(clob));
+ }
+ }
+ }
+ return aRow;
+ }
+
+ private String readFromClob(Clob clob) {
+ Reader reader = FieldReaderDataSource.readCharStream(clob);
+ StringBuilder sb = new StringBuilder();
+ char[] buf = new char[1024];
+ int len;
+ try {
+ while ((len = reader.read(buf)) != 0) {
+ sb.append(buf, 0, len);
+ }
+ } catch (IOException e) {
+ DataImportHandlerException.wrapAndThrow(DataImportHandlerException.SEVERE, e);
+ }
+ return sb.toString();
+ }
+
+ public static final String CLOB = "clob";
+}
Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ClobTransformer.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ClobTransformer.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FieldReaderDataSource.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FieldReaderDataSource.java?rev=725926&r1=725925&r2=725926&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FieldReaderDataSource.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FieldReaderDataSource.java Thu Dec 11 23:02:09 2008
@@ -16,6 +16,8 @@
*/
package org.apache.solr.handler.dataimport;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -68,14 +70,7 @@
try {
//Most of the JDBC drivers have getCharacterStream defined as public
// so let us just check it
- Method m = clob.getClass().getDeclaredMethod("getCharacterStream");
- if (Modifier.isPublic(m.getModifiers())) {
- return (Reader) m.invoke(clob);
- } else {
- // force invoke
- m.setAccessible(true);
- return (Reader) m.invoke(clob);
- }
+ return readCharStream(clob);
} catch (Exception e) {
LOG.info("Unable to get data from CLOB");
return null;
@@ -106,6 +101,22 @@
}
+ static Reader readCharStream(Clob clob) {
+ try {
+ Method m = clob.getClass().getDeclaredMethod("getCharacterStream");
+ if (Modifier.isPublic(m.getModifiers())) {
+ return (Reader) m.invoke(clob);
+ } else {
+ // force invoke
+ m.setAccessible(true);
+ return (Reader) m.invoke(clob);
+ }
+ } catch (Exception e) {
+ wrapAndThrow(SEVERE, e,"Unable to get reader from clob");
+ return null;//unreachable
+ }
+ }
+
private Reader getReader(Method m, Blob blob)
throws IllegalAccessException, InvocationTargetException, UnsupportedEncodingException {
InputStream is = (InputStream) m.invoke(blob);