You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by sh...@apache.org on 2008/07/30 21:36:00 UTC

svn commit: r681182 [1/6] - in /lucene/solr/trunk: ./ contrib/ contrib/dataimporthandler/ contrib/dataimporthandler/src/ contrib/dataimporthandler/src/main/ contrib/dataimporthandler/src/main/java/ contrib/dataimporthandler/src/main/java/org/ contrib/d...

Author: shalin
Date: Wed Jul 30 12:35:58 2008
New Revision: 681182

URL: http://svn.apache.org/viewvc?rev=681182&view=rev
Log:
SOLR-469 -- Added DataImportHandler as a contrib project.

Added:
    lucene/solr/trunk/contrib/
    lucene/solr/trunk/contrib/dataimporthandler/
    lucene/solr/trunk/contrib/dataimporthandler/build.xml   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/
    lucene/solr/trunk/contrib/dataimporthandler/src/main/
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/CachedSqlEntityProcessor.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Context.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ContextImpl.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataSource.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DateFormatTransformer.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DebugLogger.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EntityProcessor.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Evaluator.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EvaluatorBag.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FileDataSource.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FileListEntityProcessor.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/HttpDataSource.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/JdbcDataSource.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/MockDataSource.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/RegexTransformer.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ScriptTransformer.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SqlEntityProcessor.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/TemplateString.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/TemplateTransformer.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Transformer.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/VariableResolver.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/VariableResolverImpl.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathRecordReader.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/webapp/
    lucene/solr/trunk/contrib/dataimporthandler/src/main/webapp/admin/
    lucene/solr/trunk/contrib/dataimporthandler/src/main/webapp/admin/dataimport.jsp   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/main/webapp/admin/debug.jsp   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/test/
    lucene/solr/trunk/contrib/dataimporthandler/src/test/java/
    lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/
    lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/
    lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/
    lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/
    lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/
    lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestCachedSqlEntityProcessor.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDataConfig.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDateFormatTransformer.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder2.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEntityProcessorBase.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEvaluatorBag.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestJdbcDataSource.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestNumberFormatTransformer.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestRegexTransformer.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestScriptTransformer.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessor.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessor2.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestTemplateString.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestTemplateTransformer.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestVariableResolver.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestXPathRecordReader.java   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/
    lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/dataimport.properties   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/solr/
    lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/solr/conf/
    lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/solr/conf/data-config-with-datasource.xml   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/solr/conf/dataimport-nodatasource-solrconfig.xml   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/solr/conf/dataimport-schema.xml   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/solr/conf/dataimport-solrconfig.xml   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/solr/conf/dataimport.properties   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/solr/conf/protwords.txt   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/solr/conf/single-entity-data-config.xml   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/solr/conf/stopwords.txt   (with props)
    lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/solr/conf/synonyms.txt   (with props)
Modified:
    lucene/solr/trunk/CHANGES.txt

Modified: lucene/solr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/CHANGES.txt?rev=681182&r1=681181&r2=681182&view=diff
==============================================================================
--- lucene/solr/trunk/CHANGES.txt (original)
+++ lucene/solr/trunk/CHANGES.txt Wed Jul 30 12:35:58 2008
@@ -331,6 +331,11 @@
 
 65. SOLR-663: Allow multiple files for stopwords, keepwords, protwords and synonyms (shalin)
 
+66. SOLR-469: Added DataImportHandler as a contrib project which makes indexing data from Databases, XML files and HTTP
+              data sources into Solr quick and easy. Includes API and implementations for supporting multiple
+              data sources, processors and transformers for importing data. Supports full data imports as well as
+              incremental (delta) indexing. See http://wiki.apache.org/solr/DataImportHandler for more details.
+              (Noble Paul, shalin)
     
 Changes in runtime behavior
  1. SOLR-559: use Lucene updateDocument, deleteDocuments methods.  This

Added: lucene/solr/trunk/contrib/dataimporthandler/build.xml
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/build.xml?rev=681182&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/build.xml (added)
+++ lucene/solr/trunk/contrib/dataimporthandler/build.xml Wed Jul 30 12:35:58 2008
@@ -0,0 +1,129 @@
+<?xml version="1.0"?>
+
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+ 
+        http://www.apache.org/licenses/LICENSE-2.0
+ 
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+ -->
+
+<project name="solr-dataimporthandler" default="build">
+	
+  <property name="solr-path" value="../.." />
+	
+  <property name="javadoc.dir" value="target/javadoc"></property>
+	
+  <import file="../../common-build.xml"/>
+
+  <description>
+    Data Import Handler
+  </description>
+
+  <path id="common.classpath">
+  	<pathelement location="${solr-path}/build/common" />
+  	<pathelement location="${solr-path}/build/core" />
+  	<fileset dir="${solr-path}/lib" includes="*.jar"></fileset>
+  </path>
+	
+  <path id="test.classpath">
+  	<path refid="common.classpath" />
+	<pathelement path="target/classes" />
+  	<pathelement path="target/test-classes" />
+  </path>
+	
+  <target name="clean">
+  	<delete failonerror="false" dir="target"/>
+  </target>
+	
+  <target name="init">
+  	<mkdir dir="target/classes"/>
+  	<ant dir="../../" inheritall="false" target="compile" /> 
+  </target>
+	
+  <target name="compile" depends="init">
+    <solr-javac destdir="target/classes"
+                classpathref="common.classpath">
+      <src path="src/main/java" />
+    </solr-javac>
+  </target>
+	
+  <target name="build" depends="compile">
+  	<jar destfile="target/${fullnamever}.jar" basedir="target/classes" />
+  </target>
+	
+  <target name="compileTests" depends="compile">
+  	<solr-javac destdir="target/test-classes"
+  	                classpathref="test.classpath">
+  	  <src path="src/test/java" />
+  	</solr-javac>
+  </target>
+	
+  <target name="test" depends="compileTests">
+  	<mkdir dir="target/test-results"/>
+  	
+  	<junit printsummary="on"
+           haltonfailure="no"
+           errorProperty="tests.failed"
+           failureProperty="tests.failed"
+           dir="src/test/resources/"
+           >
+      <formatter type="brief" usefile="false" if="junit.details"/>
+      <classpath refid="test.classpath"/>
+      <formatter type="xml"/>
+      <batchtest fork="yes" todir="target/test-results" unless="testcase">
+        <fileset dir="src/test/java" includes="${junit.includes}"/>
+      </batchtest>
+      <batchtest fork="yes" todir="target/test-results" if="testcase">
+        <fileset dir="src/test/java" includes="**/${testcase}.java"/>
+      </batchtest>
+    </junit>
+
+    <fail if="tests.failed">Tests failed!</fail>
+  </target>
+	
+  <target name="dist" depends="build">
+  	<copy todir="../../build/web">
+  		<fileset dir="src/main/webapp" includes="**" />
+  	</copy>
+  	<mkdir dir="../../build/web/WEB-INF/lib"/>
+  	<copy file="target/${fullnamever}.jar" todir="${solr-path}/build/web/WEB-INF/lib"></copy>
+  	<copy file="target/${fullnamever}.jar" todir="${solr-path}/dist"></copy>
+  </target>
+	
+  <target name="javadoc">
+   	<sequential>
+       <mkdir dir="${javadoc.dir}/contrib-${fullnamever}"/>
+   	   <javadoc
+   		      destdir="${javadoc.dir}/contrib-${fullnamever}"
+   		      author="true"
+   		      version="true"
+   		      failonerror="true"
+   		      use="true"
+   		      encoding="utf8"
+   		      access="${javadoc.access}"
+   		      windowtitle="${Name} ${version} contrib-${fullnamever} API"
+   		      doctitle="${Name} ${version} API (${specversion})"
+   		      bottom="Copyright &amp;copy; ${javadoc.years} The Apache Software Foundation"
+   		      >
+   		        <packageset dir="src/main/java"/>
+   		        <link href="${javadoc.link.java}"/>
+   		        <link href="${javadoc.link.junit}"/>
+   		        <link href="${javadoc.link.lucene}"/>
+   		        <classpath refid="common.classpath"/>
+   		</javadoc>
+      <jar basedir="${javadoc.dir}/contrib-${fullname}" destfile="target/contrib-${fullnamever}-javadoc.jar"/>
+   	  <copy file="target/contrib-${fullnamever}-javadoc.jar" todir="${solr-path}/dist"></copy>
+    </sequential>
+  </target>	
+  
+</project>

Propchange: lucene/solr/trunk/contrib/dataimporthandler/build.xml
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: lucene/solr/trunk/contrib/dataimporthandler/build.xml
------------------------------------------------------------------------------
    svn:keywords = Date Author Id Revision HeadURL

Added: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java?rev=681182&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java (added)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java Wed Jul 30 12:35:58 2008
@@ -0,0 +1,161 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.request.LocalSolrQueryRequest;
+import org.apache.solr.util.AbstractSolrTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * <p>
+ * Abstract base class for DataImportHandler tests
+ * </p>
+ * <p/>
+ * <b>This API is experimental and subject to change</b>
+ *
+ * @version $Id$
+ * @since solr 1.3
+ */
+public abstract class AbstractDataImportHandlerTest extends
+        AbstractSolrTestCase {
+
+  @Override
+  public void setUp() throws Exception {
+    super.setUp();
+  }
+
+  @Override
+  public void tearDown() throws Exception {
+    super.tearDown();
+  }
+
+  protected String loadDataConfig(String dataConfigFileName) {
+    try {
+      SolrCore core = h.getCore();
+      return SolrWriter.getResourceAsString(core.getResourceLoader()
+              .openResource(dataConfigFileName));
+    } catch (IOException e) {
+      e.printStackTrace();
+      return null;
+    }
+  }
+
+  protected void runFullImport(String dataConfig) throws Exception {
+    LocalSolrQueryRequest request = lrf.makeRequest("command", "full-import",
+            "debug", "on", "clean", "true", "commit", "true", "dataConfig",
+            dataConfig);
+    h.query("/dataimport", request);
+  }
+
+  protected void runDeltaImport(String dataConfig) throws Exception {
+    LocalSolrQueryRequest request = lrf.makeRequest("command", "delta-import",
+            "debug", "on", "clean", "true", "commit", "true", "dataConfig",
+            dataConfig);
+    h.query("/dataimport", request);
+  }
+
+  /**
+   * Helper for creating a Context instance. Useful for testing Transformers
+   */
+  @SuppressWarnings("unchecked")
+  public static Context getContext(DataConfig.Entity parentEntity,
+                                   VariableResolverImpl resolver, DataSource parentDataSource,
+                                   int currProcess, final List<Map<String, String>> entityFields,
+                                   final Map<String, String> entityAttrs) {
+    final Context delegate = new ContextImpl(parentEntity, resolver,
+            parentDataSource, currProcess, Collections.EMPTY_MAP,
+            new HashMap<String, Object>(), null, null);
+    return new Context() {
+      public String getEntityAttribute(String name) {
+        return entityAttrs == null ? delegate.getEntityAttribute(name)
+                : entityAttrs.get(name);
+      }
+
+      public List<Map<String, String>> getAllEntityFields() {
+        return entityFields == null ? delegate.getAllEntityFields()
+                : entityFields;
+      }
+
+      public VariableResolver getVariableResolver() {
+        return delegate.getVariableResolver();
+      }
+
+      public DataSource getDataSource() {
+        return delegate.getDataSource();
+      }
+
+      public boolean isRootEntity() {
+        return false;
+      }
+
+      public int currentProcess() {
+        return delegate.currentProcess();
+      }
+
+      public Map<String, Object> getRequestParameters() {
+        return delegate.getRequestParameters();
+      }
+
+      public EntityProcessor getEntityProcessor() {
+        return null;
+      }
+
+      public void setSessionAttribute(String name, Object val, String scope) {
+        delegate.setSessionAttribute(name, val, scope);
+      }
+
+      public Object getSessionAttribute(String name, String scope) {
+        return delegate.getSessionAttribute(name, scope);
+      }
+
+      public Context getParentContext() {
+        return delegate.getParentContext();
+      }
+
+      public DataSource getDataSource(String name) {
+        return delegate.getDataSource(name);
+      }
+
+      public SolrCore getSolrCore() {
+        return delegate.getSolrCore();
+      }
+    };
+  }
+
+  /**
+   * Strings at even index are keys, odd-index strings are values in the
+   * returned map
+   */
+  @SuppressWarnings("unchecked")
+  public static Map createMap(Object... args) {
+    Map result = new HashMap();
+
+    if (args == null || args.length == 0)
+      return result;
+
+    for (int i = 0; i < args.length - 1; i += 2)
+      result.put(args[i], args[i + 1]);
+
+    return result;
+  }
+}

Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java
------------------------------------------------------------------------------
    svn:keywords = Date Author Id Revision HeadURL

Added: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/CachedSqlEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/CachedSqlEntityProcessor.java?rev=681182&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/CachedSqlEntityProcessor.java (added)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/CachedSqlEntityProcessor.java Wed Jul 30 12:35:58 2008
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * This class enables caching of data obtained from the DB to avoid too many sql
+ * queries
+ * <p/>
+ * <p>
+ * Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * </p>
+ * <p/>
+ * <b>This API is experimental and subject to change</b>
+ *
+ * @version $Id$
+ * @since solr 1.3
+ */
+public class CachedSqlEntityProcessor extends SqlEntityProcessor {
+  private boolean isFirst;
+
+  @SuppressWarnings("unchecked")
+  public void init(Context context) {
+    super.init(context);
+    super.cacheInit();
+    isFirst = true;
+  }
+
+  public Map<String, Object> nextRow() {
+    if (rowcache != null) return getFromRowCache();
+    if (dataSourceRowCache != null)
+      return getFromRowCacheTransformed();
+    if (!isFirst)
+      return null;
+    String query = resolver.replaceTokens(context.getEntityAttribute("query"));
+    isFirst = false;
+    if (simpleCache != null) {
+      return getSimplCacheData(query);
+    } else {
+      return getIdCacheData(query);
+    }
+
+  }
+
+  protected List<Map<String, Object>> getAllNonCachedRows() {
+    List<Map<String, Object>> rows = new ArrayList<Map<String, Object>>();
+    String q = getQuery();
+    initQuery(resolver.replaceTokens(q));
+    if (rowIterator == null)
+      return rows;
+    while (rowIterator.hasNext()) {
+      Map<String, Object> arow = rowIterator.next();
+      if (arow == null) {
+        break;
+      } else {
+        rows.add(arow);
+      }
+    }
+    return rows;
+  }
+}

Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/CachedSqlEntityProcessor.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/CachedSqlEntityProcessor.java
------------------------------------------------------------------------------
    svn:keywords = Date Author Id Revision HeadURL

Added: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Context.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Context.java?rev=681182&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Context.java (added)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Context.java Wed Jul 30 12:35:58 2008
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.dataimport;
+
+import org.apache.solr.core.SolrCore;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * <p>
+ * This interface is supposed to give access to all available objects. So any
+ * component implemented by a user can have the full power of DataImportHandler
+ * </p>
+ * <p>
+ * Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * </p>
+ * <p/>
+ * <b>This API is experimental and subject to change</b>
+ *
+ * @version $Id$
+ * @since solr 1.3
+ */
+public abstract class Context {
+  public static final int FULL_DUMP = 1, DELTA_DUMP = 2, FIND_DELTA = 3;
+
+  public static final String SCOPE_ENTITY = "entity", SCOPE_GLOBAL = "global",
+          SCOPE_DOC = "document";
+
+  /**
+   * Get the value of any attribute put into this entity
+   *
+   * @param name name of the attribute eg: 'name'
+   * @return value of named attribute in entity
+   */
+  public abstract String getEntityAttribute(String name);
+
+  /**
+   * Returns all the fields put into an entity. each item (which is a map ) in
+   * the list corresponds to one field. each if the map contains the attribute
+   * names and values in a field
+   *
+   * @return all fields in an entity
+   */
+  public abstract List<Map<String, String>> getAllEntityFields();
+
+  /**
+   * Returns the VariableResolver used in this entity which can be used to
+   * resolve the tokens in ${<namespce.name>}
+   *
+   * @return a VariableResolver instance
+   * @see org.apache.solr.handler.dataimport.VariableResolver
+   */
+
+  public abstract VariableResolver getVariableResolver();
+
+  /**
+   * Gets the datasource instance defined for this entity.
+   *
+   * @return a new DataSource instance as configured for the current entity
+   * @see org.apache.solr.handler.dataimport.DataSource
+   */
+  public abstract DataSource getDataSource();
+
+  /**
+   * Gets a new DataSource instance with a name.
+   *
+   * @param name Name of the dataSource as defined in the dataSource tag
+   * @return a new DataSource instance as configured for the named entity
+   * @see org.apache.solr.handler.dataimport.DataSource
+   */
+  public abstract DataSource getDataSource(String name);
+
+  /**
+   * Returns the instance of EntityProcessor used for this entity
+   *
+   * @return instance of EntityProcessor used for the current entity
+   * @see org.apache.solr.handler.dataimport.EntityProcessor
+   */
+  public abstract EntityProcessor getEntityProcessor();
+
+  /**
+   * Store values in a certain name and scope (entity, document,global)
+   *
+   * @param name  the key
+   * @param val   the value
+   * @param scope the scope in which the given key, value pair is to be stored
+   */
+  public abstract void setSessionAttribute(String name, Object val, String scope);
+
+  /**
+   * get a value by name in the given scope (entity, document,global)
+   *
+   * @param name  the key
+   * @param scope the scope from which the value is to be retreived
+   * @return the object stored in the given scope with the given key
+   */
+  public abstract Object getSessionAttribute(String name, String scope);
+
+  /**
+   * Get the context instance for the parent entity. works only in the full dump
+   * If the current entity is rootmost a null is returned
+   *
+   * @return parent entity's Context
+   */
+  public abstract Context getParentContext();
+
+  /**
+   * The request parameters passed over HTTP for this command the values in the
+   * map are either String(for single valued parameters) or List<String> (for
+   * multi-valued parameters)
+   *
+   * @return the request parameters passed in the URL to initiate this process
+   */
+  public abstract Map<String, Object> getRequestParameters();
+
+  /**
+   * Returns if the current entity is the root entity
+   *
+   * @return true if current entity is the root entity, false otherwise
+   */
+  public abstract boolean isRootEntity();
+
+  /**
+   * Returns the current process FULL_DUMP =1, DELTA_DUMP=2, FIND_DELTA=3
+   *
+   * @return the code of the current running process
+   */
+  public abstract int currentProcess();
+
+  /**
+   * Exposing the actual SolrCore to the components
+   *
+   * @return the core
+   */
+  public abstract SolrCore getSolrCore();
+}

Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Context.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Context.java
------------------------------------------------------------------------------
    svn:keywords = Date Author Id Revision HeadURL

Added: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ContextImpl.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ContextImpl.java?rev=681182&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ContextImpl.java (added)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ContextImpl.java Wed Jul 30 12:35:58 2008
@@ -0,0 +1,143 @@
+package org.apache.solr.handler.dataimport;
+
+import org.apache.solr.core.SolrCore;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * <p>
+ * An implementation for the Context
+ * </p>
+ * <p/>
+ * <b>This API is experimental and subject to change</b>
+ *
+ * @version $Id$
+ * @since solr 1.3
+ */
+public class ContextImpl extends Context {
+  private DataConfig.Entity entity;
+
+  private ContextImpl parent;
+
+  private VariableResolverImpl resolver;
+
+  private DataSource ds;
+
+  private int currProcess;
+
+  private Map<String, Object> requestParams;
+
+  private DataImporter dataImporter;
+
+  private Map<String, Object> entitySession, globalSession, docSession;
+
+  public ContextImpl(DataConfig.Entity entity, VariableResolverImpl resolver,
+                     DataSource ds, int currProcess, Map<String, Object> requestParams,
+                     Map<String, Object> global, ContextImpl p, DataImporter di) {
+    this.entity = entity;
+    this.resolver = resolver;
+    this.ds = ds;
+    this.currProcess = currProcess;
+    this.requestParams = requestParams;
+    globalSession = global;
+    parent = p;
+    dataImporter = di;
+  }
+
+  public String getEntityAttribute(String name) {
+    return entity == null ? null : entity.allAttributes.get(name);
+  }
+
+  public List<Map<String, String>> getAllEntityFields() {
+    return entity == null ? Collections.EMPTY_LIST : entity.allFieldsList;
+  }
+
+  public VariableResolver getVariableResolver() {
+    return resolver;
+  }
+
+  public DataSource getDataSource() {
+    return ds;
+  }
+
+  public DataSource getDataSource(String name) {
+    return dataImporter.getDataSourceInstance(entity);
+  }
+
+  public boolean isRootEntity() {
+    return entity.isDocRoot;
+  }
+
+  public int currentProcess() {
+    return currProcess;
+  }
+
+  public Map<String, Object> getRequestParameters() {
+    return requestParams;
+  }
+
+  public EntityProcessor getEntityProcessor() {
+    return entity == null ? null : entity.processor;
+  }
+
+  public void setSessionAttribute(String name, Object val, String scope) {
+    if (Context.SCOPE_ENTITY.equals(scope)) {
+      if (entitySession == null)
+        entitySession = new HashMap<String, Object>();
+      entitySession.put(name, val);
+    } else if (Context.SCOPE_GLOBAL.equals(scope)) {
+      if (globalSession != null) {
+        globalSession.put(name, val);
+      }
+    } else if (Context.SCOPE_DOC.equals(scope)) {
+      Map<String, Object> docsession = getDocSession();
+      if (docsession != null)
+        docsession.put(name, val);
+    }
+  }
+
+  public Object getSessionAttribute(String name, String scope) {
+    if (Context.SCOPE_ENTITY.equals(scope)) {
+      if (entitySession == null)
+        return null;
+      return entitySession.get(name);
+    } else if (Context.SCOPE_GLOBAL.equals(scope)) {
+      if (globalSession != null) {
+        return globalSession.get(name);
+      }
+    } else if (Context.SCOPE_DOC.equals(scope)) {
+      Map<String, Object> docsession = getDocSession();
+      if (docsession != null)
+        return docsession.get(name);
+    }
+    return null;
+  }
+
+  public Context getParentContext() {
+    return parent;
+  }
+
+  public Map<String, Object> getDocSession() {
+    ContextImpl c = this;
+    while (true) {
+      if (c.docSession != null)
+        return c.docSession;
+      if (c.parent != null)
+        c = c.parent;
+      else
+        return null;
+    }
+  }
+
+  public void setDocSession(Map<String, Object> docSession) {
+    this.docSession = docSession;
+  }
+
+
+  public SolrCore getSolrCore() {
+    return dataImporter.getCore();
+  }
+}

Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ContextImpl.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ContextImpl.java
------------------------------------------------------------------------------
    svn:keywords = Date Author Id Revision HeadURL

Added: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java?rev=681182&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java (added)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java Wed Jul 30 12:35:58 2008
@@ -0,0 +1,360 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import java.util.*;
+
+/**
+ * <p>
+ * Mapping for data-config.xml
+ * </p>
+ * <p/>
+ * <p>
+ * Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * </p>
+ * <p/>
+ * <b>This API is experimental and subject to change</b>
+ *
+ * @version $Id$
+ * @since solr 1.3
+ */
+public class DataConfig {
+  public List<Document> documents;
+
+  public List<Props> properties;
+
+  private Map<String, Document> documentCache;
+
+  public Map<String, Evaluator> evaluators = new HashMap<String, Evaluator>();
+
+  public Script script;
+
+  public Map<String, Properties> dataSources = new HashMap<String, Properties>();
+
+  public Document getDocumentByName(String name) {
+    if (documentCache == null) {
+      documentCache = new HashMap<String, Document>();
+      for (Document document : documents)
+        documentCache.put(document.name, document);
+    }
+
+    return documentCache.get(name);
+  }
+
+  public static class Document {
+    public String name;
+
+    public String deleteQuery;
+
+    public List<Entity> entities = new ArrayList<Entity>();
+
+    public List<Field> fields;
+
+    public Document() {
+    }
+
+    public Document(Element element) {
+      this.name = getStringAttribute(element, NAME, null);
+      this.deleteQuery = getStringAttribute(element, "deleteQuery", null);
+      List<Element> l = getChildNodes(element, "entity");
+      for (Element e : l)
+        entities.add(new Entity(e));
+      // entities = new Entity(l.get(0));
+      l = getChildNodes(element, "field");
+      if (!l.isEmpty())
+        fields = new ArrayList<Field>();
+      for (Element e : l)
+        fields.add(new Field(e));
+    }
+  }
+
+  public static class Props {
+    public String name;
+
+    public String file;
+  }
+
+  public static class Entity {
+    public String name;
+
+    public String pk;
+
+    public String dataSource;
+
+    public Map<String, String> allAttributes;
+
+    public String proc;
+
+    public String docRoot;
+
+    public boolean isDocRoot = false;
+
+    public List<Field> fields;
+
+    public List<Map<String, String>> allFieldsList = new ArrayList<Map<String, String>>();
+
+    public List<Entity> entities;
+
+    public String[] primaryKeys;
+
+    public Entity parentEntity;
+
+    public EntityProcessor processor;
+
+    @SuppressWarnings("unchecked")
+    public DataSource dataSrc;
+
+    public Script script;
+
+    public List<Field> implicitFields;
+
+    public Entity() {
+    }
+
+    public Entity(Element element) {
+      name = getStringAttribute(element, NAME, null);
+      pk = getStringAttribute(element, "pk", null);
+      docRoot = getStringAttribute(element, ROOT_ENTITY, null);
+      proc = getStringAttribute(element, PROCESSOR, null);
+      dataSource = getStringAttribute(element, DataImporter.DATA_SRC, null);
+      allAttributes = getAllAttributes(element);
+      List<Element> n = getChildNodes(element, "field");
+      fields = new ArrayList<Field>();
+      for (Element elem : n)
+        fields.add(new Field(elem));
+      n = getChildNodes(element, "entity");
+      if (!n.isEmpty())
+        entities = new ArrayList<Entity>();
+      for (Element elem : n)
+        entities.add(new Entity(elem));
+
+    }
+
+    public void clearCache() {
+      if (entities != null) {
+        for (Entity entity : entities)
+          entity.clearCache();
+      }
+
+      try {
+        processor.destroy();
+      } catch (Exception e) {
+        /*no op*/
+      }
+      processor = null;
+      if (dataSrc != null)
+        dataSrc.close();
+
+    }
+  }
+
+  public static class Script {
+    public String language;
+
+    public String script;
+
+    public Script() {
+    }
+
+    public Script(Element e) {
+      this.language = getStringAttribute(e, "language", "JavaScript");
+      StringBuffer buffer = new StringBuffer();
+      String script = getTxt(e, buffer);
+      if (script != null)
+        this.script = script.trim();
+    }
+  }
+
+  public static class Field {
+
+    public String column;
+
+    public String name;
+
+    public Float boost = 1.0f;
+
+    public boolean toWrite = true;
+
+    public boolean multiValued = false;
+
+    public String nameOrColName;
+
+    public Map<String, String> allAttributes = new HashMap<String, String>() {
+      public String put(String key, String value) {
+        if (super.containsKey(key))
+          return super.get(key);
+        return super.put(key, value);
+      }
+    };
+
+    public Field() {
+    }
+
+    public Field(Element e) {
+      this.name = getStringAttribute(e, DataImporter.NAME, null);
+      this.column = getStringAttribute(e, DataImporter.COLUMN, null);
+      this.boost = Float.parseFloat(getStringAttribute(e, "boost", "1.0f"));
+      allAttributes.putAll(getAllAttributes(e));
+    }
+
+    public Field(String name, boolean b) {
+      name = nameOrColName = column = name;
+      multiValued = b;
+
+    }
+
+    public String getName() {
+      return name == null ? column : name;
+    }
+
+    public Entity entity;
+
+  }
+
+  public void readFromXml(Element e) {
+    List<Element> n = getChildNodes(e, "document");
+    if (!n.isEmpty())
+      documents = new ArrayList<Document>();
+    for (Element element : n)
+      documents.add(new Document(element));
+
+    n = getChildNodes(e, SCRIPT);
+    if (!n.isEmpty()) {
+      script = new Script(n.get(0));
+    }
+
+    // Add the provided evaluators
+    evaluators.put(EvaluatorBag.DATE_FORMAT_EVALUATOR, EvaluatorBag
+            .getDateFormatEvaluator());
+    evaluators.put(EvaluatorBag.SQL_ESCAPE_EVALUATOR, EvaluatorBag
+            .getSqlEscapingEvaluator());
+    evaluators.put(EvaluatorBag.URL_ENCODE_EVALUATOR, EvaluatorBag
+            .getUrlEvaluator());
+
+    n = getChildNodes(e, FUNCTION);
+    if (!n.isEmpty()) {
+      for (Element element : n) {
+        String func = getStringAttribute(element, NAME, null);
+        String clz = getStringAttribute(element, CLASS, null);
+        if (func == null || clz == null)
+          throw new DataImportHandlerException(
+                  DataImportHandlerException.SEVERE,
+                  "<function> must have a 'name' and 'class' attributes");
+        try {
+          evaluators.put(func, (Evaluator) DocBuilder.loadClass(clz)
+                  .newInstance());
+        } catch (Exception exp) {
+          throw new DataImportHandlerException(
+                  DataImportHandlerException.SEVERE,
+                  "Unable to instantiate evaluator: " + clz, exp);
+        }
+      }
+    }
+    n = getChildNodes(e, DATA_SRC);
+    if (!n.isEmpty()) {
+      for (Element element : n) {
+        Properties p = new Properties();
+        HashMap<String, String> attrs = getAllAttributes(element);
+        for (Map.Entry<String, String> entry : attrs.entrySet()) {
+          p.setProperty(entry.getKey(), entry.getValue());
+        }
+        dataSources.put(p.getProperty("name"), p);
+      }
+    }
+  }
+
+  private static String getStringAttribute(Element e, String name, String def) {
+    String r = e.getAttribute(name);
+    if (r == null || "".equals(r.trim()))
+      r = def;
+    return r;
+  }
+
+  private static HashMap<String, String> getAllAttributes(Element e) {
+    HashMap<String, String> m = new HashMap<String, String>();
+    NamedNodeMap nnm = e.getAttributes();
+    for (int i = 0; i < nnm.getLength(); i++) {
+      m.put(nnm.item(i).getNodeName(), nnm.item(i).getNodeValue());
+    }
+    return m;
+  }
+
+  public static String getTxt(Node elem, StringBuffer buffer) {
+
+    if (elem.getNodeType() != Node.CDATA_SECTION_NODE) {
+      NodeList childs = elem.getChildNodes();
+      for (int i = 0; i < childs.getLength(); i++) {
+        Node child = childs.item(i);
+        short childType = child.getNodeType();
+        if (childType != Node.COMMENT_NODE
+                && childType != Node.PROCESSING_INSTRUCTION_NODE) {
+          getTxt(child, buffer);
+        }
+      }
+    } else {
+      buffer.append(elem.getNodeValue());
+    }
+
+    return buffer.toString();
+  }
+
+  public static List<Element> getChildNodes(Element e, String byName) {
+    List<Element> result = new ArrayList<Element>();
+    NodeList l = e.getChildNodes();
+    for (int i = 0; i < l.getLength(); i++) {
+      if (e.equals(l.item(i).getParentNode())
+              && byName.equals(l.item(i).getNodeName()))
+        result.add((Element) l.item(i));
+    }
+    return result;
+  }
+
+  public void clearCaches() {
+    for (Document document : documents)
+      for (Entity entity : document.entities)
+        entity.clearCache();
+
+  }
+
+  public static final String SCRIPT = "script";
+
+  public static final String NAME = "name";
+
+  public static final String SCRIPT_LANG = "scriptlanguage";
+
+  public static final String SCRIPT_NAME = "scriptname";
+
+  public static final String PROCESSOR = "processor";
+
+  public static final String IMPORTER_NS = "dataimporter";
+
+  public static final String ROOT_ENTITY = "rootEntity";
+
+  public static final String FUNCTION = "function";
+
+  public static final String CLASS = "class";
+
+  public static final String DATA_SRC = "dataSource";
+
+}

Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java
------------------------------------------------------------------------------
    svn:keywords = Date Author Id Revision HeadURL

Added: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java?rev=681182&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java (added)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java Wed Jul 30 12:35:58 2008
@@ -0,0 +1,394 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.apache.lucene.document.Document;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStreamBase;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.SolrConfig;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.SolrResourceLoader;
+import org.apache.solr.handler.RequestHandlerBase;
+import org.apache.solr.handler.RequestHandlerUtils;
+import org.apache.solr.request.RawResponseWriter;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrQueryResponse;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.update.DocumentBuilder;
+import org.apache.solr.update.UpdateHandler;
+import org.apache.solr.util.plugin.SolrCoreAware;
+
+import java.util.*;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * <p>
+ * Solr Request Handler for data import from databases and REST data sources.
+ * </p>
+ * <p>
+ * It is configured in solrconfig.xml
+ * </p>
+ * <p/>
+ * <p>
+ * Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * </p>
+ * <p/>
+ * <b>This API is experimental and subject to change</b>
+ *
+ * @version $Id$
+ * @since solr 1.3
+ */
+public class DataImportHandler extends RequestHandlerBase implements
+        SolrCoreAware {
+
+  private static final Logger LOG = Logger.getLogger(DataImportHandler.class
+          .getName());
+
+  private DataImporter importer;
+
+  private Map<String, String> variables = new HashMap<String, String>();
+
+  @SuppressWarnings("unchecked")
+  private NamedList initArgs;
+
+  private Map<String, Properties> dataSources = new HashMap<String, Properties>();
+
+  private DataImporter.RequestParams requestParams;
+
+  private List<Document> debugDocuments;
+
+  private DebugLogger debugLogger;
+
+  private boolean debugEnabled = true;
+
+  @Override
+  @SuppressWarnings("unchecked")
+  public void init(NamedList args) {
+    super.init(args);
+
+    initArgs = args;
+  }
+
+  @SuppressWarnings("unchecked")
+  public void inform(SolrCore core) {
+    try {
+      String debug = (String) initArgs.get(ENABLE_DEBUG);
+      if (debug != null && "no".equals(debug))
+        debugEnabled = false;
+      NamedList defaults = (NamedList) initArgs.get("defaults");
+      if (defaults != null) {
+        String configLoc = (String) defaults.get("config");
+        if (configLoc != null && configLoc.length() != 0) {
+          processConfiguration(defaults);
+
+          importer = new DataImporter(SolrWriter.getResourceAsString(core
+                  .getResourceLoader().openResource(configLoc)), core,
+                  dataSources);
+        }
+      }
+    } catch (Throwable e) {
+      SolrConfig.severeErrors.add(e);
+      LOG.log(Level.SEVERE, DataImporter.MSG.LOAD_EXP, e);
+      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+              DataImporter.MSG.INVALID_CONFIG, e);
+    }
+  }
+
+  @Override
+  @SuppressWarnings("unchecked")
+  public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp)
+          throws Exception {
+    rsp.setHttpCaching(false);
+    SolrParams params = req.getParams();
+    requestParams = new DataImporter.RequestParams(getParamsMap(params));
+    String command = requestParams.command;
+
+    if (DataImporter.SHOW_CONF_CMD.equals(command)) {
+      // Modify incoming request params to add wt=raw
+      ModifiableSolrParams rawParams = new ModifiableSolrParams(req.getParams());
+      rawParams.set(CommonParams.WT, "raw");
+      req.setParams(rawParams);
+      String dataConfigFile = defaults.get("config");
+      ContentStreamBase content = new ContentStreamBase.StringStream(SolrWriter
+              .getResourceAsString(req.getCore().getResourceLoader().openResource(
+              dataConfigFile)));
+      rsp.add(RawResponseWriter.CONTENT, content);
+      return;
+    }
+
+    rsp.add("initArgs", initArgs);
+    String message = "";
+
+    if (command != null)
+      rsp.add("command", command);
+
+    if (requestParams.debug) {
+      // Reload the data-config.xml
+      importer = null;
+      if (requestParams.dataConfig != null) {
+        try {
+          processConfiguration((NamedList) initArgs.get("defaults"));
+          importer = new DataImporter(requestParams.dataConfig, req.getCore()
+                  , dataSources);
+        } catch (RuntimeException e) {
+          rsp.add("exception", DebugLogger.getStacktraceString(e));
+          importer = null;
+          return;
+        }
+      } else {
+        inform(req.getCore());
+      }
+      message = DataImporter.MSG.CONFIG_RELOADED;
+    }
+
+    // If importer is still null
+    if (importer == null) {
+      rsp.add("status", DataImporter.MSG.NO_INIT);
+      return;
+    }
+
+    if (command != null && DataImporter.ABORT_CMD.equals(command)) {
+      importer.rumCmd(requestParams, null, null);
+    } else if (importer.getStatus() != DataImporter.Status.IDLE) {
+      message = DataImporter.MSG.CMD_RUNNING;
+    } else if (command != null) {
+      if (DataImporter.FULL_IMPORT_CMD.equals(command)
+              || DataImporter.DELTA_IMPORT_CMD.equals(command)) {
+        UpdateHandler updater = req.getCore().getUpdateHandler();
+        SolrResourceLoader loader = req.getCore().getResourceLoader();
+        SolrWriter sw = getSolrWriter(updater, loader, req
+                .getSchema());
+
+        if (requestParams.debug) {
+          if (debugEnabled) {
+            // Synchronous request for the debug mode
+            importer.rumCmd(requestParams, sw, variables);
+            rsp.add("mode", "debug");
+            rsp.add("documents", debugDocuments);
+            if (debugLogger != null)
+              rsp.add("verbose-output", debugLogger.output);
+            debugLogger = null;
+            debugDocuments = null;
+          } else {
+            message = DataImporter.MSG.DEBUG_NOT_ENABLED;
+          }
+        } else {
+          // Asynchronous request for normal mode
+          importer.runAsync(requestParams, sw, variables);
+        }
+      } else if (DataImporter.RELOAD_CONF_CMD.equals(command)) {
+        importer = null;
+        inform(req.getCore());
+        message = DataImporter.MSG.CONFIG_RELOADED;
+      }
+    }
+    rsp.add("status", importer.getStatus() == DataImporter.Status.IDLE ? "idle"
+            : "busy");
+    rsp.add("importResponse", message);
+    rsp.add("statusMessages", importer.getStatusMessages());
+
+    RequestHandlerUtils.addExperimentalFormatWarning(rsp);
+  }
+
+  private Map<String, Object> getParamsMap(SolrParams params) {
+    Iterator<String> names = params.getParameterNamesIterator();
+    Map<String, Object> result = new HashMap<String, Object>();
+    while (names.hasNext()) {
+      String s = names.next();
+      String[] val = params.getParams(s);
+      if (val == null || val.length < 1)
+        continue;
+      if (val.length == 1)
+        result.put(s, val[0]);
+      else
+        result.put(s, Arrays.asList(val));
+    }
+    return result;
+  }
+
+  @SuppressWarnings("unchecked")
+  private void processConfiguration(NamedList defaults) {
+    if (defaults == null) {
+      LOG
+              .info("No configuration specified in solrconfig.xml for DataImportHandler");
+      return;
+    }
+
+    LOG.info("Processing configuration from solrconfig.xml: " + defaults);
+
+    dataSources = new HashMap<String, Properties>();
+    variables = new HashMap<String, String>();
+
+    int position = 0;
+
+    while (position < defaults.size()) {
+      if (defaults.getName(position) == null)
+        break;
+
+      String name = defaults.getName(position);
+      if (name.equals("datasource")) {
+        NamedList dsConfig = (NamedList) defaults.getVal(position);
+        Properties props = new Properties();
+        for (int i = 0; i < dsConfig.size(); i++)
+          props.put(dsConfig.getName(i), dsConfig.getVal(i));
+        LOG.info("Adding properties to datasource: " + props);
+        dataSources.put((String) dsConfig.get("name"), props);
+      } else if (!name.equals("config")) {
+        String value = (String) defaults.getVal(position);
+        variables.put(name, value);
+      }
+      position++;
+    }
+  }
+
+  private SolrWriter getSolrWriter(final UpdateHandler updater,
+                                   final SolrResourceLoader loader, final IndexSchema schema) {
+
+    return new SolrWriter(updater, loader.getConfigDir()) {
+
+      @Override
+      public boolean upload(SolrDoc d) {
+        try {
+          Document document = DocumentBuilder.toDocument(
+                  ((SolrDocumentWrapper) d).doc, schema);
+          if (requestParams.debug) {
+            if (debugDocuments == null)
+              debugDocuments = new ArrayList<Document>();
+            debugDocuments.add(document);
+            if (debugDocuments.size() >= requestParams.rows) {
+              // Abort this operation now
+              importer.getDocBuilder().abort();
+            }
+          }
+
+          return super.upload(document);
+        } catch (RuntimeException e) {
+          LOG.log(Level.SEVERE, "Exception while adding: " + d, e);
+          return false;
+        }
+      }
+
+      public void log(int event, String name, Object row) {
+        if (debugLogger == null) {
+          debugLogger = new DebugLogger();
+        }
+        debugLogger.log(event, name, row);
+      }
+
+
+      public Class loadClass(String name) throws ClassNotFoundException {
+        return loader.findClass(name);
+      }
+
+      public SolrDoc getSolrDocInstance() {
+        return new SolrDocumentWrapper();
+      }
+    };
+  }
+
+  static class SolrDocumentWrapper implements SolrWriter.SolrDoc {
+    SolrInputDocument doc;
+
+    public SolrDocumentWrapper() {
+      doc = new SolrInputDocument();
+    }
+
+    public void setDocumentBoost(float boost) {
+      doc.setDocumentBoost(boost);
+    }
+
+    public Object getField(String field) {
+      return doc.getField(field);
+    }
+
+    public void addField(String name, Object value, float boost) {
+      doc.addField(name, value, boost);
+    }
+
+    public String toString() {
+      return doc.toString();
+    }
+  }
+
+  @Override
+  @SuppressWarnings("unchecked")
+  public NamedList getStatistics() {
+    if (importer == null)
+      return super.getStatistics();
+
+    DocBuilder.Statistics cumulative = importer.cumulativeStatistics;
+    NamedList result = new NamedList();
+
+    result.add("Status", importer.getStatus().toString());
+
+    if (importer.docBuilder != null) {
+      DocBuilder.Statistics running = importer.docBuilder.importStatistics;
+      result.add("Documents Processed", running.docCount);
+      result.add("Requests made to DataSource", running.queryCount);
+      result.add("Rows Fetched", running.rowsCount);
+      result.add("Documents Deleted", running.deletedDocCount);
+      result.add("Documents Skipped", running.skipDocCount);
+    }
+
+    result.add(DataImporter.MSG.TOTAL_DOC_PROCESSED, cumulative.docCount);
+    result.add(DataImporter.MSG.TOTAL_QUERIES_EXECUTED, cumulative.queryCount);
+    result.add(DataImporter.MSG.TOTAL_ROWS_EXECUTED, cumulative.rowsCount);
+    result.add(DataImporter.MSG.TOTAL_DOCS_DELETED, cumulative.deletedDocCount);
+    result.add(DataImporter.MSG.TOTAL_DOCS_SKIPPED, cumulative.skipDocCount);
+
+    NamedList requestStatistics = super.getStatistics();
+    if (requestStatistics != null) {
+      for (int i = 0; i < requestStatistics.size(); i++) {
+        result.add(requestStatistics.getName(i), requestStatistics.getVal(i));
+      }
+    }
+
+    return result;
+  }
+
+  // //////////////////////SolrInfoMBeans methods //////////////////////
+
+  @Override
+  public String getDescription() {
+    return DataImporter.MSG.JMX_DESC;
+  }
+
+  @Override
+  public String getSourceId() {
+    return "$Id$";
+  }
+
+  @Override
+  public String getVersion() {
+    return "1.0";
+  }
+
+  @Override
+  public String getSource() {
+    return "$URL$";
+  }
+
+  public static final String ENABLE_DEBUG = "enableDebug";
+}

Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java
------------------------------------------------------------------------------
    svn:keywords = Date Author Id Revision HeadURL

Added: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java?rev=681182&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java (added)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java Wed Jul 30 12:35:58 2008
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.dataimport;
+
+/**
+ * <p>
+ * Exception class for all DataImportHandler exceptions
+ * </p>
+ * <p/>
+ * <b>This API is experimental and subject to change</b>
+ * <p/>
+ * $Id$
+ *
+ * @since solr 1.3
+ */
+public class DataImportHandlerException extends RuntimeException {
+  private int errCode;
+
+  public boolean debugged = false;
+
+  public static final int SEVERE = 500, WARN = 400, SKIP = 300;
+
+  public DataImportHandlerException(int err) {
+    super();
+    errCode = err;
+  }
+
+  public DataImportHandlerException(int err, String message) {
+    super(message + MSG + SolrWriter.getDocCount());
+    errCode = err;
+  }
+
+  public DataImportHandlerException(int err, String message, Throwable cause) {
+    super(message + MSG + SolrWriter.getDocCount(), cause);
+    errCode = err;
+  }
+
+  public DataImportHandlerException(int err, Throwable cause) {
+    super(cause);
+    errCode = err;
+  }
+
+  public int getErrCode() {
+    return errCode;
+  }
+
+  public static final String MSG = " Processing Documemt # ";
+}

Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java
------------------------------------------------------------------------------
    svn:keywords = Date Author Id Revision HeadURL

Added: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java?rev=681182&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java (added)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java Wed Jul 30 12:35:58 2008
@@ -0,0 +1,544 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.dataimport;
+
+import org.apache.solr.core.SolrConfig;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaField;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.xml.sax.InputSource;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import java.io.StringReader;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * <p>
+ * Stores all configuration information for pulling and indexing data.
+ * </p>
+ * <p/>
+ * <b>This API is experimental and subject to change</b>
+ *
+ * @version $Id$
+ * @since solr 1.3
+ */
+public class DataImporter {
+
+  public enum Status {
+    IDLE, RUNNING_FULL_DUMP, RUNNING_DELTA_DUMP, JOB_FAILED
+  }
+
+  private static final Logger LOG = Logger.getLogger(DataImporter.class
+          .getName());
+
+  private Status status = Status.IDLE;
+
+  private DataConfig config;
+
+  private Date lastIndexTime;
+
+  private Date indexStartTime;
+
+  private Properties store = new Properties();
+
+  private Map<String, Properties> dataSourceProps;
+
+  private IndexSchema schema;
+
+  public DocBuilder docBuilder;
+
+  public DocBuilder.Statistics cumulativeStatistics = new DocBuilder.Statistics();
+
+  public Map<String, Evaluator> evaluators;
+
+  private SolrCore core;
+
+  /**
+   * Only for testing purposes
+   */
+  DataImporter() {
+  }
+
+  public DataImporter(String dataConfig, SolrCore core,
+                      Map<String, Properties> ds) {
+    if (dataConfig == null)
+      throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+              "Configuration not found");
+    this.core = core;
+    this.schema = core.getSchema();
+    dataSourceProps = ds;
+    loadDataConfig(dataConfig);
+
+    for (DataConfig.Document document : config.documents) {
+      for (DataConfig.Entity e : document.entities) {
+        Map<String, DataConfig.Field> fields = new HashMap<String, DataConfig.Field>();
+        initEntity(e, fields, false);
+        e.implicitFields = new ArrayList<DataConfig.Field>();
+        String errs = verifyWithSchema(fields, e.implicitFields);
+        if (e.implicitFields.isEmpty())
+          e.implicitFields = null;
+        if (errs != null) {
+          throw new DataImportHandlerException(
+                  DataImportHandlerException.SEVERE, errs);
+        }
+      }
+    }
+  }
+
+  private String verifyWithSchema(Map<String, DataConfig.Field> fields,
+                                  List<DataConfig.Field> autoFields) {
+    List<String> errors = new ArrayList<String>();
+    Map<String, SchemaField> schemaFields = schema.getFields();
+    for (Map.Entry<String, SchemaField> entry : schemaFields.entrySet()) {
+      SchemaField sf = entry.getValue();
+      if (!fields.containsKey(sf.getName())) {
+        if (sf.isRequired()) {
+          LOG
+                  .info(sf.getName()
+                          + " is a required field in SolrSchema . But not found in DataConfig");
+        }
+        autoFields.add(new DataConfig.Field(sf.getName(), sf.multiValued()));
+      }
+    }
+    for (Map.Entry<String, DataConfig.Field> entry : fields.entrySet()) {
+      DataConfig.Field fld = entry.getValue();
+      FieldType fieldType = null;
+
+      try {
+        fieldType = schema.getDynamicFieldType(fld.name);
+      } catch (RuntimeException e) {
+        // Ignore because it may not be a dynamic field
+      }
+
+      if (fld.name != null) {
+        if (schema.getFields().get(fld.name) == null && fieldType == null) {
+          errors
+                  .add("The field :"
+                          + fld.name
+                          + " present in DataConfig does not have a counterpart in Solr Schema");
+        }
+      } else if (schema.getFields().get(fld.column) == null
+              && fieldType == null) {
+        LOG.info("Column : " + fld.column + " is not a schema field");
+      }
+    }
+
+    if (!errors.isEmpty()) {
+      StringBuffer sb = new StringBuffer("There are errors in the Schema\n");
+      for (String error : errors) {
+        sb.append(error).append("\n");
+      }
+      return sb.toString();
+
+    }
+    return null;
+
+  }
+
+  void loadDataConfig(String configFile) {
+
+    try {
+      DocumentBuilder builder = DocumentBuilderFactory.newInstance()
+              .newDocumentBuilder();
+      Document document = builder.parse(new InputSource(new StringReader(
+              configFile)));
+
+      config = new DataConfig();
+      config.readFromXml((Element) document.getElementsByTagName("dataConfig")
+              .item(0));
+
+      LOG.info("Data Configuration loaded successfully");
+    } catch (Exception e) {
+      SolrConfig.severeErrors.add(e);
+      throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+              "Exception occurred while initializing context", e);
+    }
+  }
+
+  private void initEntity(DataConfig.Entity e,
+                          Map<String, DataConfig.Field> fields, boolean docRootFound) {
+    if (e.pk != null)
+      e.primaryKeys = e.pk.split(",");
+    e.allAttributes.put(DATA_SRC, e.dataSource);
+
+    if (!docRootFound && !"false".equals(e.docRoot)) {
+      // if in this chain no document root is found()
+      e.isDocRoot = true;
+    }
+
+    if (e.fields != null) {
+      for (DataConfig.Field f : e.fields) {
+        f.nameOrColName = f.getName();
+        SchemaField schemaField = schema.getFields().get(f.getName());
+        if (schemaField != null) {
+          f.multiValued = schemaField.multiValued();
+          f.allAttributes.put(MULTI_VALUED, Boolean.toString(schemaField
+                  .multiValued()));
+          f.allAttributes.put(TYPE, schemaField.getType().getTypeName());
+          f.allAttributes.put("indexed", Boolean
+                  .toString(schemaField.indexed()));
+          f.allAttributes.put("stored", Boolean.toString(schemaField.stored()));
+          f.allAttributes.put("defaultValue", schemaField.getDefaultValue());
+        } else {
+
+          try {
+            f.allAttributes.put(TYPE, schema.getDynamicFieldType(f.getName())
+                    .getTypeName());
+            f.allAttributes.put(MULTI_VALUED, "true");
+            f.multiValued = true;
+          } catch (RuntimeException e2) {
+            LOG.info("Field in data-config.xml - " + f.getName()
+                    + " not found in schema.xml");
+            f.toWrite = false;
+          }
+        }
+        fields.put(f.getName(), f);
+        f.entity = e;
+        f.allAttributes.put("boost", f.boost.toString());
+        f.allAttributes.put("toWrite", Boolean.toString(f.toWrite));
+        e.allFieldsList.add(Collections.unmodifiableMap(f.allAttributes));
+      }
+    }
+    e.allFieldsList = Collections.unmodifiableList(e.allFieldsList);
+    e.allAttributes = Collections.unmodifiableMap(e.allAttributes);
+
+    addDataSource(e);
+
+    if (e.entities == null)
+      return;
+    for (DataConfig.Entity e1 : e.entities) {
+      e1.parentEntity = e;
+      initEntity(e1, fields, e.isDocRoot || docRootFound);
+    }
+
+  }
+
+  public DataConfig getConfig() {
+    return config;
+  }
+
+  public Date getIndexStartTime() {
+    return indexStartTime;
+  }
+
+  public void setIndexStartTime(Date indextStartTime) {
+    this.indexStartTime = indextStartTime;
+  }
+
+  public Date getLastIndexTime() {
+    return lastIndexTime;
+  }
+
+  public void setLastIndexTime(Date lastIndexTime) {
+    this.lastIndexTime = lastIndexTime;
+  }
+
+  public void store(Object key, Object value) {
+    store.put(key, value);
+  }
+
+  public Object retrieve(Object key) {
+    return store.get(key);
+  }
+
+  @SuppressWarnings("unchecked")
+  public void addDataSource(DataConfig.Entity key) {
+    if ("null".equals(key.dataSource)) {
+      key.dataSrc = new MockDataSource();
+      return;
+    }
+    key.dataSrc = getDataSourceInstance(key);
+  }
+
+  DataSource getDataSourceInstance(DataConfig.Entity key) {
+    Properties p = dataSourceProps.get(key.dataSource);
+    if (p == null)
+      p = config.dataSources.get(key.dataSource);
+    if (p == null)
+      p = dataSourceProps.get(null);// for default data source
+    if (p == null)
+      p = config.dataSources.get(null);
+    if (p == null)
+      throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+              "No dataSource :" + key.dataSource + " available for entity :"
+                      + key.name);
+    String impl = p.getProperty(TYPE);
+    DataSource dataSrc = null;
+    if (impl == null) {
+      dataSrc = new JdbcDataSource();
+    } else {
+      try {
+        dataSrc = (DataSource) DocBuilder.loadClass(impl).newInstance();
+      } catch (Exception e) {
+        throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+                "Invalid type for data source: " + impl, e);
+      }
+    }
+    try {
+      Properties copyProps = new Properties();
+      copyProps.putAll(p);
+      dataSrc.init(new ContextImpl(key, null, dataSrc, 0,
+              Collections.EMPTY_MAP, new HashMap(), null, this), copyProps);
+    } catch (Exception e) {
+      throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+              "Failed to initialize DataSource: " + key.dataSource, e);
+    }
+    return dataSrc;
+  }
+
+  public Status getStatus() {
+    return status;
+  }
+
+  public void setStatus(Status status) {
+    this.status = status;
+  }
+
+  public void doFullImport(SolrWriter writer, RequestParams requestParams,
+                           Map<String, String> variables) {
+    LOG.info("Starting Full Import");
+    setStatus(Status.RUNNING_FULL_DUMP);
+
+    if (requestParams.commit)
+      setIndexStartTime(new Date());
+
+    try {
+      if (requestParams.clean)
+        writer.doDeleteAll();
+      docBuilder = new DocBuilder(this, writer, requestParams, variables);
+      docBuilder.execute(getConfig().documents.get(0).name);
+      if (!requestParams.debug)
+        cumulativeStatistics.add(docBuilder.importStatistics);
+    } catch (RuntimeException e) {
+      LOG.log(Level.SEVERE, "Full Import failed", e);
+    } finally {
+      setStatus(Status.IDLE);
+      config.clearCaches();
+      DocBuilder.INSTANCE.set(null);
+    }
+
+  }
+
+  public void doDeltaImport(SolrWriter writer, RequestParams requestParams,
+                            Map<String, String> variables) {
+    LOG.info("Starting Delta Import");
+    setStatus(Status.RUNNING_DELTA_DUMP);
+
+    try {
+      if (requestParams.commit) {
+        Date lastModified = writer.loadIndexStartTime();
+        setIndexStartTime(new Date());
+        setLastIndexTime(lastModified);
+      }
+      docBuilder = new DocBuilder(this, writer, requestParams, variables);
+      docBuilder.execute(config.documents.get(0).name);
+      if (!requestParams.debug)
+        cumulativeStatistics.add(docBuilder.importStatistics);
+    } catch (RuntimeException e) {
+      LOG.log(Level.SEVERE, "Delta Import Failed", e);
+    } finally {
+      setStatus(Status.IDLE);
+      config.clearCaches();
+      DocBuilder.INSTANCE.set(null);
+    }
+
+  }
+
+  public void runAsync(final RequestParams reqParams, final SolrWriter sw,
+                       final Map<String, String> variables) {
+    new Thread() {
+      @Override
+      public void run() {
+        rumCmd(reqParams, sw, variables);
+      }
+    }.start();
+  }
+
+  void rumCmd(RequestParams reqParams, SolrWriter sw,
+              Map<String, String> variables) {
+    String command = reqParams.command;
+    if (command.equals("full-import")) {
+      doFullImport(sw, reqParams, variables);
+    } else if (command.equals(DELTA_IMPORT_CMD)) {
+      doDeltaImport(sw, reqParams, variables);
+    } else if (command.equals(ABORT_CMD)) {
+      if (docBuilder != null)
+        docBuilder.abort();
+    }
+  }
+
+  @SuppressWarnings("unchecked")
+  Map<String, String> getStatusMessages() {
+    Map statusMessages = (Map) retrieve(STATUS_MSGS);
+    Map<String, String> result = new LinkedHashMap<String, String>();
+    if (statusMessages != null) {
+      for (Object o : statusMessages.entrySet()) {
+        Map.Entry e = (Map.Entry) o;
+        result.put((String) e.getKey(), e.getValue().toString());
+      }
+    }
+    return result;
+
+  }
+
+  public DocBuilder getDocBuilder() {
+    return docBuilder;
+  }
+
+  public static final ThreadLocal<AtomicLong> QUERY_COUNT = new ThreadLocal<AtomicLong>() {
+    protected AtomicLong initialValue() {
+      return new AtomicLong();
+    }
+  };
+
+  static final SimpleDateFormat DATE_TIME_FORMAT = new SimpleDateFormat(
+          "yyyy-MM-dd HH:mm:ss");
+
+  static final class MSG {
+    public static final String NO_CONFIG_FOUND = "Configuration not found";
+
+    public static final String NO_INIT = "DataImportHandler started. Not Initialized. No commands can be run";
+
+    public static final String INVALID_CONFIG = "FATAL: Could not create importer. DataImporter config invalid";
+
+    public static final String LOAD_EXP = "Exception while loading DataImporter";
+
+    public static final String JMX_DESC = "Manage data import from databases to Solr";
+
+    public static final String CMD_RUNNING = "A command is still running...";
+
+    public static final String DEBUG_NOT_ENABLED = "Debug not enabled. Add a tag <str name=\"enableDebug\">true</str> in solrconfig.xml";
+
+    public static final String CONFIG_RELOADED = "Configuration Re-loaded sucessfully";
+
+    public static final String TOTAL_DOC_PROCESSED = "Total Documents Processed";
+
+    public static final String TOTAL_QUERIES_EXECUTED = "Total Requests made to DataSource";
+
+    public static final String TOTAL_ROWS_EXECUTED = "Total Rows Fetched";
+
+    public static final String TOTAL_DOCS_DELETED = "Total Documents Deleted";
+
+    public static final String TOTAL_DOCS_SKIPPED = "Total Documents Skipped";
+  }
+
+  static final class RequestParams {
+    public String command = null;
+
+    public boolean debug = false;
+
+    public boolean verbose = false;
+
+    public boolean commit = true;
+
+    public boolean optimize = true;
+
+    public int start = 0;
+
+    public int rows = 10;
+
+    public boolean clean = true;
+
+    public List<String> entities;
+
+    public Map<String, Object> requestParams;
+
+    public String dataConfig;
+
+    public RequestParams() {
+    }
+
+    public RequestParams(Map<String, Object> requestParams) {
+      if (requestParams.containsKey("command"))
+        command = (String) requestParams.get("command");
+
+      if ("on".equals(requestParams.get("debug"))) {
+        debug = true;
+        // Set default values suitable for debug mode
+        commit = false;
+        clean = false;
+        verbose = "true".equals(requestParams.get("verbose"))
+                || "on".equals(requestParams.get("verbose"));
+      }
+      if (requestParams.containsKey("commit"))
+        commit = Boolean.parseBoolean((String) requestParams.get("commit"));
+      if (requestParams.containsKey("start"))
+        start = Integer.parseInt((String) requestParams.get("start"));
+      if (requestParams.containsKey("rows"))
+        rows = Integer.parseInt((String) requestParams.get("rows"));
+      if (requestParams.containsKey("clean"))
+        clean = Boolean.parseBoolean((String) requestParams.get("clean"));
+      if (requestParams.containsKey("optimize"))
+        optimize = Boolean.parseBoolean((String) requestParams.get("optimize"));
+
+      Object o = requestParams.get("entity");
+
+      if (o != null && o instanceof String) {
+        entities = new ArrayList<String>();
+        entities.add((String) o);
+      } else if (o != null && o instanceof List) {
+        entities = (List<String>) requestParams.get("entity");
+      }
+
+      dataConfig = (String) requestParams.get("dataConfig");
+      if (dataConfig != null && dataConfig.trim().length() == 0) {
+        // Empty data-config param is not valid, change it to null
+        dataConfig = null;
+      }
+
+      this.requestParams = requestParams;
+    }
+  }
+
+
+  public SolrCore getCore() {
+    return core;
+  }
+
+  public static final String COLUMN = "column";
+
+  public static final String TYPE = "type";
+
+  public static final String DATA_SRC = "dataSource";
+
+  public static final String MULTI_VALUED = "multiValued";
+
+  public static final String NAME = "name";
+
+  public static final String STATUS_MSGS = "status-messages";
+
+  public static final String FULL_IMPORT_CMD = "full-import";
+
+  public static final String DELTA_IMPORT_CMD = "delta-import";
+
+  public static final String ABORT_CMD = "abort";
+
+  public static final String DEBUG_MODE = "debug";
+
+  public static final String RELOAD_CONF_CMD = "reload-config";
+
+  public static final String SHOW_CONF_CMD = "show-config";
+}

Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
------------------------------------------------------------------------------
    svn:keywords = Date Author Id Revision HeadURL

Added: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataSource.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataSource.java?rev=681182&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataSource.java (added)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataSource.java Wed Jul 30 12:35:58 2008
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.dataimport;
+
+import java.util.Properties;
+
+/**
+ * <p>
+ * Provides data from a source with a given query.
+ * </p>
+ * <p/>
+ * <p>
+ * Implementation of this interface must provide a default no-arg constructor
+ * </p>
+ * <p/>
+ * <p>
+ * Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * </p>
+ * <p/>
+ * <b>This API is experimental and may change in the future.</b>
+ *
+ * @version $Id$
+ * @since solr 1.3
+ */
+public abstract class DataSource<T> {
+
+  /**
+   * Initializes the DataSource with the <code>Context</code> and
+   * initialization properties.
+   * <p/>
+   * This is invoked by the <code>DataImporter</code> after creating an
+   * instance of this class.
+   *
+   * @param context
+   * @param initProps
+   */
+  public abstract void init(Context context, Properties initProps);
+
+  /**
+   * Get records for the given query.The return type depends on the
+   * implementation .
+   *
+   * @param query The query string. It can be a SQL for JdbcDataSource or a URL
+   *              for HttpDataSource or a file location for FileDataSource or a custom
+   *              format for your own custom DataSource.
+   * @return Depends on the implementation. For instance JdbcDataSource returns
+   *         an Iterator<Map <String,Object>>
+   */
+  public abstract T getData(String query);
+
+  /**
+   * Cleans up resources of this DataSource after use.
+   */
+  public abstract void close();
+}

Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataSource.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataSource.java
------------------------------------------------------------------------------
    svn:keywords = Date Author Id Revision HeadURL