You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by sh...@apache.org on 2008/07/30 21:36:00 UTC
svn commit: r681182 [1/6] - in /lucene/solr/trunk: ./ contrib/
contrib/dataimporthandler/ contrib/dataimporthandler/src/
contrib/dataimporthandler/src/main/ contrib/dataimporthandler/src/main/java/
contrib/dataimporthandler/src/main/java/org/ contrib/d...
Author: shalin
Date: Wed Jul 30 12:35:58 2008
New Revision: 681182
URL: http://svn.apache.org/viewvc?rev=681182&view=rev
Log:
SOLR-469 -- Added DataImportHandler as a contrib project.
Added:
lucene/solr/trunk/contrib/
lucene/solr/trunk/contrib/dataimporthandler/
lucene/solr/trunk/contrib/dataimporthandler/build.xml (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/
lucene/solr/trunk/contrib/dataimporthandler/src/main/
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/CachedSqlEntityProcessor.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Context.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ContextImpl.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataSource.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DateFormatTransformer.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DebugLogger.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EntityProcessor.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EntityProcessorBase.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Evaluator.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/EvaluatorBag.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FileDataSource.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FileListEntityProcessor.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/HttpDataSource.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/JdbcDataSource.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/MockDataSource.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/RegexTransformer.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ScriptTransformer.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SolrWriter.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/SqlEntityProcessor.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/TemplateString.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/TemplateTransformer.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Transformer.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/VariableResolver.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/VariableResolverImpl.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathRecordReader.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/webapp/
lucene/solr/trunk/contrib/dataimporthandler/src/main/webapp/admin/
lucene/solr/trunk/contrib/dataimporthandler/src/main/webapp/admin/dataimport.jsp (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/main/webapp/admin/debug.jsp (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/test/
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestCachedSqlEntityProcessor.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDataConfig.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDateFormatTransformer.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder2.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEntityProcessorBase.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEvaluatorBag.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestJdbcDataSource.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestNumberFormatTransformer.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestRegexTransformer.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestScriptTransformer.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessor.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessor2.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestTemplateString.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestTemplateTransformer.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestVariableResolver.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestXPathRecordReader.java (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/
lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/dataimport.properties (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/solr/
lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/solr/conf/
lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/solr/conf/data-config-with-datasource.xml (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/solr/conf/dataimport-nodatasource-solrconfig.xml (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/solr/conf/dataimport-schema.xml (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/solr/conf/dataimport-solrconfig.xml (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/solr/conf/dataimport.properties (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/solr/conf/protwords.txt (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/solr/conf/single-entity-data-config.xml (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/solr/conf/stopwords.txt (with props)
lucene/solr/trunk/contrib/dataimporthandler/src/test/resources/solr/conf/synonyms.txt (with props)
Modified:
lucene/solr/trunk/CHANGES.txt
Modified: lucene/solr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/CHANGES.txt?rev=681182&r1=681181&r2=681182&view=diff
==============================================================================
--- lucene/solr/trunk/CHANGES.txt (original)
+++ lucene/solr/trunk/CHANGES.txt Wed Jul 30 12:35:58 2008
@@ -331,6 +331,11 @@
65. SOLR-663: Allow multiple files for stopwords, keepwords, protwords and synonyms (shalin)
+66. SOLR-469: Added DataImportHandler as a contrib project which makes indexing data from Databases, XML files and HTTP
+ data sources into Solr quick and easy. Includes API and implementations for supporting multiple
+ data sources, processors and transformers for importing data. Supports full data imports as well as
+ incremental (delta) indexing. See http://wiki.apache.org/solr/DataImportHandler for more details.
+ (Noble Paul, shalin)
Changes in runtime behavior
1. SOLR-559: use Lucene updateDocument, deleteDocuments methods. This
Added: lucene/solr/trunk/contrib/dataimporthandler/build.xml
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/build.xml?rev=681182&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/build.xml (added)
+++ lucene/solr/trunk/contrib/dataimporthandler/build.xml Wed Jul 30 12:35:58 2008
@@ -0,0 +1,129 @@
+<?xml version="1.0"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<project name="solr-dataimporthandler" default="build">
+
+ <property name="solr-path" value="../.." />
+
+ <property name="javadoc.dir" value="target/javadoc"></property>
+
+ <import file="../../common-build.xml"/>
+
+ <description>
+ Data Import Handler
+ </description>
+
+ <path id="common.classpath">
+ <pathelement location="${solr-path}/build/common" />
+ <pathelement location="${solr-path}/build/core" />
+ <fileset dir="${solr-path}/lib" includes="*.jar"></fileset>
+ </path>
+
+ <path id="test.classpath">
+ <path refid="common.classpath" />
+ <pathelement path="target/classes" />
+ <pathelement path="target/test-classes" />
+ </path>
+
+ <target name="clean">
+ <delete failonerror="false" dir="target"/>
+ </target>
+
+ <target name="init">
+ <mkdir dir="target/classes"/>
+ <ant dir="../../" inheritall="false" target="compile" />
+ </target>
+
+ <target name="compile" depends="init">
+ <solr-javac destdir="target/classes"
+ classpathref="common.classpath">
+ <src path="src/main/java" />
+ </solr-javac>
+ </target>
+
+ <target name="build" depends="compile">
+ <jar destfile="target/${fullnamever}.jar" basedir="target/classes" />
+ </target>
+
+ <target name="compileTests" depends="compile">
+ <solr-javac destdir="target/test-classes"
+ classpathref="test.classpath">
+ <src path="src/test/java" />
+ </solr-javac>
+ </target>
+
+ <target name="test" depends="compileTests">
+ <mkdir dir="target/test-results"/>
+
+ <junit printsummary="on"
+ haltonfailure="no"
+ errorProperty="tests.failed"
+ failureProperty="tests.failed"
+ dir="src/test/resources/"
+ >
+ <formatter type="brief" usefile="false" if="junit.details"/>
+ <classpath refid="test.classpath"/>
+ <formatter type="xml"/>
+ <batchtest fork="yes" todir="target/test-results" unless="testcase">
+ <fileset dir="src/test/java" includes="${junit.includes}"/>
+ </batchtest>
+ <batchtest fork="yes" todir="target/test-results" if="testcase">
+ <fileset dir="src/test/java" includes="**/${testcase}.java"/>
+ </batchtest>
+ </junit>
+
+ <fail if="tests.failed">Tests failed!</fail>
+ </target>
+
+ <target name="dist" depends="build">
+ <copy todir="../../build/web">
+ <fileset dir="src/main/webapp" includes="**" />
+ </copy>
+ <mkdir dir="../../build/web/WEB-INF/lib"/>
+ <copy file="target/${fullnamever}.jar" todir="${solr-path}/build/web/WEB-INF/lib"></copy>
+ <copy file="target/${fullnamever}.jar" todir="${solr-path}/dist"></copy>
+ </target>
+
+ <target name="javadoc">
+ <sequential>
+ <mkdir dir="${javadoc.dir}/contrib-${fullnamever}"/>
+ <javadoc
+ destdir="${javadoc.dir}/contrib-${fullnamever}"
+ author="true"
+ version="true"
+ failonerror="true"
+ use="true"
+ encoding="utf8"
+ access="${javadoc.access}"
+ windowtitle="${Name} ${version} contrib-${fullnamever} API"
+ doctitle="${Name} ${version} API (${specversion})"
+ bottom="Copyright &copy; ${javadoc.years} The Apache Software Foundation"
+ >
+ <packageset dir="src/main/java"/>
+ <link href="${javadoc.link.java}"/>
+ <link href="${javadoc.link.junit}"/>
+ <link href="${javadoc.link.lucene}"/>
+ <classpath refid="common.classpath"/>
+ </javadoc>
+ <jar basedir="${javadoc.dir}/contrib-${fullname}" destfile="target/contrib-${fullnamever}-javadoc.jar"/>
+ <copy file="target/contrib-${fullnamever}-javadoc.jar" todir="${solr-path}/dist"></copy>
+ </sequential>
+ </target>
+
+</project>
Propchange: lucene/solr/trunk/contrib/dataimporthandler/build.xml
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/solr/trunk/contrib/dataimporthandler/build.xml
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Added: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java?rev=681182&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java (added)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java Wed Jul 30 12:35:58 2008
@@ -0,0 +1,161 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.request.LocalSolrQueryRequest;
+import org.apache.solr.util.AbstractSolrTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * <p>
+ * Abstract base class for DataImportHandler tests
+ * </p>
+ * <p/>
+ * <b>This API is experimental and subject to change</b>
+ *
+ * @version $Id$
+ * @since solr 1.3
+ */
+public abstract class AbstractDataImportHandlerTest extends
+ AbstractSolrTestCase {
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ super.tearDown();
+ }
+
+ protected String loadDataConfig(String dataConfigFileName) {
+ try {
+ SolrCore core = h.getCore();
+ return SolrWriter.getResourceAsString(core.getResourceLoader()
+ .openResource(dataConfigFileName));
+ } catch (IOException e) {
+ e.printStackTrace();
+ return null;
+ }
+ }
+
+ protected void runFullImport(String dataConfig) throws Exception {
+ LocalSolrQueryRequest request = lrf.makeRequest("command", "full-import",
+ "debug", "on", "clean", "true", "commit", "true", "dataConfig",
+ dataConfig);
+ h.query("/dataimport", request);
+ }
+
+ protected void runDeltaImport(String dataConfig) throws Exception {
+ LocalSolrQueryRequest request = lrf.makeRequest("command", "delta-import",
+ "debug", "on", "clean", "true", "commit", "true", "dataConfig",
+ dataConfig);
+ h.query("/dataimport", request);
+ }
+
+ /**
+ * Helper for creating a Context instance. Useful for testing Transformers
+ */
+ @SuppressWarnings("unchecked")
+ public static Context getContext(DataConfig.Entity parentEntity,
+ VariableResolverImpl resolver, DataSource parentDataSource,
+ int currProcess, final List<Map<String, String>> entityFields,
+ final Map<String, String> entityAttrs) {
+ final Context delegate = new ContextImpl(parentEntity, resolver,
+ parentDataSource, currProcess, Collections.EMPTY_MAP,
+ new HashMap<String, Object>(), null, null);
+ return new Context() {
+ public String getEntityAttribute(String name) {
+ return entityAttrs == null ? delegate.getEntityAttribute(name)
+ : entityAttrs.get(name);
+ }
+
+ public List<Map<String, String>> getAllEntityFields() {
+ return entityFields == null ? delegate.getAllEntityFields()
+ : entityFields;
+ }
+
+ public VariableResolver getVariableResolver() {
+ return delegate.getVariableResolver();
+ }
+
+ public DataSource getDataSource() {
+ return delegate.getDataSource();
+ }
+
+ public boolean isRootEntity() {
+ return false;
+ }
+
+ public int currentProcess() {
+ return delegate.currentProcess();
+ }
+
+ public Map<String, Object> getRequestParameters() {
+ return delegate.getRequestParameters();
+ }
+
+ public EntityProcessor getEntityProcessor() {
+ return null;
+ }
+
+ public void setSessionAttribute(String name, Object val, String scope) {
+ delegate.setSessionAttribute(name, val, scope);
+ }
+
+ public Object getSessionAttribute(String name, String scope) {
+ return delegate.getSessionAttribute(name, scope);
+ }
+
+ public Context getParentContext() {
+ return delegate.getParentContext();
+ }
+
+ public DataSource getDataSource(String name) {
+ return delegate.getDataSource(name);
+ }
+
+ public SolrCore getSolrCore() {
+ return delegate.getSolrCore();
+ }
+ };
+ }
+
+ /**
+ * Strings at even index are keys, odd-index strings are values in the
+ * returned map
+ */
+ @SuppressWarnings("unchecked")
+ public static Map createMap(Object... args) {
+ Map result = new HashMap();
+
+ if (args == null || args.length == 0)
+ return result;
+
+ for (int i = 0; i < args.length - 1; i += 2)
+ result.put(args[i], args[i + 1]);
+
+ return result;
+ }
+}
Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Added: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/CachedSqlEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/CachedSqlEntityProcessor.java?rev=681182&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/CachedSqlEntityProcessor.java (added)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/CachedSqlEntityProcessor.java Wed Jul 30 12:35:58 2008
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * This class enables caching of data obtained from the DB to avoid too many sql
+ * queries
+ * <p/>
+ * <p>
+ * Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * </p>
+ * <p/>
+ * <b>This API is experimental and subject to change</b>
+ *
+ * @version $Id$
+ * @since solr 1.3
+ */
+public class CachedSqlEntityProcessor extends SqlEntityProcessor {
+ private boolean isFirst;
+
+ @SuppressWarnings("unchecked")
+ public void init(Context context) {
+ super.init(context);
+ super.cacheInit();
+ isFirst = true;
+ }
+
+ public Map<String, Object> nextRow() {
+ if (rowcache != null) return getFromRowCache();
+ if (dataSourceRowCache != null)
+ return getFromRowCacheTransformed();
+ if (!isFirst)
+ return null;
+ String query = resolver.replaceTokens(context.getEntityAttribute("query"));
+ isFirst = false;
+ if (simpleCache != null) {
+ return getSimplCacheData(query);
+ } else {
+ return getIdCacheData(query);
+ }
+
+ }
+
+ protected List<Map<String, Object>> getAllNonCachedRows() {
+ List<Map<String, Object>> rows = new ArrayList<Map<String, Object>>();
+ String q = getQuery();
+ initQuery(resolver.replaceTokens(q));
+ if (rowIterator == null)
+ return rows;
+ while (rowIterator.hasNext()) {
+ Map<String, Object> arow = rowIterator.next();
+ if (arow == null) {
+ break;
+ } else {
+ rows.add(arow);
+ }
+ }
+ return rows;
+ }
+}
Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/CachedSqlEntityProcessor.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/CachedSqlEntityProcessor.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Added: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Context.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Context.java?rev=681182&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Context.java (added)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Context.java Wed Jul 30 12:35:58 2008
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.dataimport;
+
+import org.apache.solr.core.SolrCore;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * <p>
+ * This interface is supposed to give access to all available objects. So any
+ * component implemented by a user can have the full power of DataImportHandler
+ * </p>
+ * <p>
+ * Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * </p>
+ * <p/>
+ * <b>This API is experimental and subject to change</b>
+ *
+ * @version $Id$
+ * @since solr 1.3
+ */
+public abstract class Context {
+ public static final int FULL_DUMP = 1, DELTA_DUMP = 2, FIND_DELTA = 3;
+
+ public static final String SCOPE_ENTITY = "entity", SCOPE_GLOBAL = "global",
+ SCOPE_DOC = "document";
+
+ /**
+ * Get the value of any attribute put into this entity
+ *
+ * @param name name of the attribute eg: 'name'
+ * @return value of named attribute in entity
+ */
+ public abstract String getEntityAttribute(String name);
+
+ /**
+ * Returns all the fields put into an entity. each item (which is a map ) in
+ * the list corresponds to one field. each if the map contains the attribute
+ * names and values in a field
+ *
+ * @return all fields in an entity
+ */
+ public abstract List<Map<String, String>> getAllEntityFields();
+
+ /**
+ * Returns the VariableResolver used in this entity which can be used to
+ * resolve the tokens in ${<namespce.name>}
+ *
+ * @return a VariableResolver instance
+ * @see org.apache.solr.handler.dataimport.VariableResolver
+ */
+
+ public abstract VariableResolver getVariableResolver();
+
+ /**
+ * Gets the datasource instance defined for this entity.
+ *
+ * @return a new DataSource instance as configured for the current entity
+ * @see org.apache.solr.handler.dataimport.DataSource
+ */
+ public abstract DataSource getDataSource();
+
+ /**
+ * Gets a new DataSource instance with a name.
+ *
+ * @param name Name of the dataSource as defined in the dataSource tag
+ * @return a new DataSource instance as configured for the named entity
+ * @see org.apache.solr.handler.dataimport.DataSource
+ */
+ public abstract DataSource getDataSource(String name);
+
+ /**
+ * Returns the instance of EntityProcessor used for this entity
+ *
+ * @return instance of EntityProcessor used for the current entity
+ * @see org.apache.solr.handler.dataimport.EntityProcessor
+ */
+ public abstract EntityProcessor getEntityProcessor();
+
+ /**
+ * Store values in a certain name and scope (entity, document,global)
+ *
+ * @param name the key
+ * @param val the value
+ * @param scope the scope in which the given key, value pair is to be stored
+ */
+ public abstract void setSessionAttribute(String name, Object val, String scope);
+
+ /**
+ * get a value by name in the given scope (entity, document,global)
+ *
+ * @param name the key
+ * @param scope the scope from which the value is to be retreived
+ * @return the object stored in the given scope with the given key
+ */
+ public abstract Object getSessionAttribute(String name, String scope);
+
+ /**
+ * Get the context instance for the parent entity. works only in the full dump
+ * If the current entity is rootmost a null is returned
+ *
+ * @return parent entity's Context
+ */
+ public abstract Context getParentContext();
+
+ /**
+ * The request parameters passed over HTTP for this command the values in the
+ * map are either String(for single valued parameters) or List<String> (for
+ * multi-valued parameters)
+ *
+ * @return the request parameters passed in the URL to initiate this process
+ */
+ public abstract Map<String, Object> getRequestParameters();
+
+ /**
+ * Returns if the current entity is the root entity
+ *
+ * @return true if current entity is the root entity, false otherwise
+ */
+ public abstract boolean isRootEntity();
+
+ /**
+ * Returns the current process FULL_DUMP =1, DELTA_DUMP=2, FIND_DELTA=3
+ *
+ * @return the code of the current running process
+ */
+ public abstract int currentProcess();
+
+ /**
+ * Exposing the actual SolrCore to the components
+ *
+ * @return the core
+ */
+ public abstract SolrCore getSolrCore();
+}
Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Context.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/Context.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Added: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ContextImpl.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ContextImpl.java?rev=681182&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ContextImpl.java (added)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ContextImpl.java Wed Jul 30 12:35:58 2008
@@ -0,0 +1,143 @@
+package org.apache.solr.handler.dataimport;
+
+import org.apache.solr.core.SolrCore;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * <p>
+ * An implementation for the Context
+ * </p>
+ * <p/>
+ * <b>This API is experimental and subject to change</b>
+ *
+ * @version $Id$
+ * @since solr 1.3
+ */
+public class ContextImpl extends Context {
+ private DataConfig.Entity entity;
+
+ private ContextImpl parent;
+
+ private VariableResolverImpl resolver;
+
+ private DataSource ds;
+
+ private int currProcess;
+
+ private Map<String, Object> requestParams;
+
+ private DataImporter dataImporter;
+
+ private Map<String, Object> entitySession, globalSession, docSession;
+
+ public ContextImpl(DataConfig.Entity entity, VariableResolverImpl resolver,
+ DataSource ds, int currProcess, Map<String, Object> requestParams,
+ Map<String, Object> global, ContextImpl p, DataImporter di) {
+ this.entity = entity;
+ this.resolver = resolver;
+ this.ds = ds;
+ this.currProcess = currProcess;
+ this.requestParams = requestParams;
+ globalSession = global;
+ parent = p;
+ dataImporter = di;
+ }
+
+ public String getEntityAttribute(String name) {
+ return entity == null ? null : entity.allAttributes.get(name);
+ }
+
+ public List<Map<String, String>> getAllEntityFields() {
+ return entity == null ? Collections.EMPTY_LIST : entity.allFieldsList;
+ }
+
+ public VariableResolver getVariableResolver() {
+ return resolver;
+ }
+
+ public DataSource getDataSource() {
+ return ds;
+ }
+
+ public DataSource getDataSource(String name) {
+ return dataImporter.getDataSourceInstance(entity);
+ }
+
+ public boolean isRootEntity() {
+ return entity.isDocRoot;
+ }
+
+ public int currentProcess() {
+ return currProcess;
+ }
+
+ public Map<String, Object> getRequestParameters() {
+ return requestParams;
+ }
+
+ public EntityProcessor getEntityProcessor() {
+ return entity == null ? null : entity.processor;
+ }
+
+ public void setSessionAttribute(String name, Object val, String scope) {
+ if (Context.SCOPE_ENTITY.equals(scope)) {
+ if (entitySession == null)
+ entitySession = new HashMap<String, Object>();
+ entitySession.put(name, val);
+ } else if (Context.SCOPE_GLOBAL.equals(scope)) {
+ if (globalSession != null) {
+ globalSession.put(name, val);
+ }
+ } else if (Context.SCOPE_DOC.equals(scope)) {
+ Map<String, Object> docsession = getDocSession();
+ if (docsession != null)
+ docsession.put(name, val);
+ }
+ }
+
+ public Object getSessionAttribute(String name, String scope) {
+ if (Context.SCOPE_ENTITY.equals(scope)) {
+ if (entitySession == null)
+ return null;
+ return entitySession.get(name);
+ } else if (Context.SCOPE_GLOBAL.equals(scope)) {
+ if (globalSession != null) {
+ return globalSession.get(name);
+ }
+ } else if (Context.SCOPE_DOC.equals(scope)) {
+ Map<String, Object> docsession = getDocSession();
+ if (docsession != null)
+ return docsession.get(name);
+ }
+ return null;
+ }
+
+ public Context getParentContext() {
+ return parent;
+ }
+
+ public Map<String, Object> getDocSession() {
+ ContextImpl c = this;
+ while (true) {
+ if (c.docSession != null)
+ return c.docSession;
+ if (c.parent != null)
+ c = c.parent;
+ else
+ return null;
+ }
+ }
+
+ public void setDocSession(Map<String, Object> docSession) {
+ this.docSession = docSession;
+ }
+
+
+ public SolrCore getSolrCore() {
+ return dataImporter.getCore();
+ }
+}
Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ContextImpl.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ContextImpl.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Added: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java?rev=681182&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java (added)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java Wed Jul 30 12:35:58 2008
@@ -0,0 +1,360 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import java.util.*;
+
+/**
+ * <p>
+ * Mapping for data-config.xml
+ * </p>
+ * <p/>
+ * <p>
+ * Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * </p>
+ * <p/>
+ * <b>This API is experimental and subject to change</b>
+ *
+ * @version $Id$
+ * @since solr 1.3
+ */
+public class DataConfig {
+ public List<Document> documents;
+
+ public List<Props> properties;
+
+ private Map<String, Document> documentCache;
+
+ public Map<String, Evaluator> evaluators = new HashMap<String, Evaluator>();
+
+ public Script script;
+
+ public Map<String, Properties> dataSources = new HashMap<String, Properties>();
+
+ public Document getDocumentByName(String name) {
+ if (documentCache == null) {
+ documentCache = new HashMap<String, Document>();
+ for (Document document : documents)
+ documentCache.put(document.name, document);
+ }
+
+ return documentCache.get(name);
+ }
+
+ public static class Document {
+ public String name;
+
+ public String deleteQuery;
+
+ public List<Entity> entities = new ArrayList<Entity>();
+
+ public List<Field> fields;
+
+ public Document() {
+ }
+
+ public Document(Element element) {
+ this.name = getStringAttribute(element, NAME, null);
+ this.deleteQuery = getStringAttribute(element, "deleteQuery", null);
+ List<Element> l = getChildNodes(element, "entity");
+ for (Element e : l)
+ entities.add(new Entity(e));
+ // entities = new Entity(l.get(0));
+ l = getChildNodes(element, "field");
+ if (!l.isEmpty())
+ fields = new ArrayList<Field>();
+ for (Element e : l)
+ fields.add(new Field(e));
+ }
+ }
+
+ public static class Props {
+ public String name;
+
+ public String file;
+ }
+
+ public static class Entity {
+ public String name;
+
+ public String pk;
+
+ public String dataSource;
+
+ public Map<String, String> allAttributes;
+
+ public String proc;
+
+ public String docRoot;
+
+ public boolean isDocRoot = false;
+
+ public List<Field> fields;
+
+ public List<Map<String, String>> allFieldsList = new ArrayList<Map<String, String>>();
+
+ public List<Entity> entities;
+
+ public String[] primaryKeys;
+
+ public Entity parentEntity;
+
+ public EntityProcessor processor;
+
+ @SuppressWarnings("unchecked")
+ public DataSource dataSrc;
+
+ public Script script;
+
+ public List<Field> implicitFields;
+
+ public Entity() {
+ }
+
+ public Entity(Element element) {
+ name = getStringAttribute(element, NAME, null);
+ pk = getStringAttribute(element, "pk", null);
+ docRoot = getStringAttribute(element, ROOT_ENTITY, null);
+ proc = getStringAttribute(element, PROCESSOR, null);
+ dataSource = getStringAttribute(element, DataImporter.DATA_SRC, null);
+ allAttributes = getAllAttributes(element);
+ List<Element> n = getChildNodes(element, "field");
+ fields = new ArrayList<Field>();
+ for (Element elem : n)
+ fields.add(new Field(elem));
+ n = getChildNodes(element, "entity");
+ if (!n.isEmpty())
+ entities = new ArrayList<Entity>();
+ for (Element elem : n)
+ entities.add(new Entity(elem));
+
+ }
+
+ public void clearCache() {
+ if (entities != null) {
+ for (Entity entity : entities)
+ entity.clearCache();
+ }
+
+ try {
+ processor.destroy();
+ } catch (Exception e) {
+ /*no op*/
+ }
+ processor = null;
+ if (dataSrc != null)
+ dataSrc.close();
+
+ }
+ }
+
+ public static class Script {
+ public String language;
+
+ public String script;
+
+ public Script() {
+ }
+
+ public Script(Element e) {
+ this.language = getStringAttribute(e, "language", "JavaScript");
+ StringBuffer buffer = new StringBuffer();
+ String script = getTxt(e, buffer);
+ if (script != null)
+ this.script = script.trim();
+ }
+ }
+
+ public static class Field {
+
+ public String column;
+
+ public String name;
+
+ public Float boost = 1.0f;
+
+ public boolean toWrite = true;
+
+ public boolean multiValued = false;
+
+ public String nameOrColName;
+
+ public Map<String, String> allAttributes = new HashMap<String, String>() {
+ public String put(String key, String value) {
+ if (super.containsKey(key))
+ return super.get(key);
+ return super.put(key, value);
+ }
+ };
+
+ public Field() {
+ }
+
+ public Field(Element e) {
+ this.name = getStringAttribute(e, DataImporter.NAME, null);
+ this.column = getStringAttribute(e, DataImporter.COLUMN, null);
+ this.boost = Float.parseFloat(getStringAttribute(e, "boost", "1.0f"));
+ allAttributes.putAll(getAllAttributes(e));
+ }
+
+ public Field(String name, boolean b) {
+ name = nameOrColName = column = name;
+ multiValued = b;
+
+ }
+
+ public String getName() {
+ return name == null ? column : name;
+ }
+
+ public Entity entity;
+
+ }
+
+ public void readFromXml(Element e) {
+ List<Element> n = getChildNodes(e, "document");
+ if (!n.isEmpty())
+ documents = new ArrayList<Document>();
+ for (Element element : n)
+ documents.add(new Document(element));
+
+ n = getChildNodes(e, SCRIPT);
+ if (!n.isEmpty()) {
+ script = new Script(n.get(0));
+ }
+
+ // Add the provided evaluators
+ evaluators.put(EvaluatorBag.DATE_FORMAT_EVALUATOR, EvaluatorBag
+ .getDateFormatEvaluator());
+ evaluators.put(EvaluatorBag.SQL_ESCAPE_EVALUATOR, EvaluatorBag
+ .getSqlEscapingEvaluator());
+ evaluators.put(EvaluatorBag.URL_ENCODE_EVALUATOR, EvaluatorBag
+ .getUrlEvaluator());
+
+ n = getChildNodes(e, FUNCTION);
+ if (!n.isEmpty()) {
+ for (Element element : n) {
+ String func = getStringAttribute(element, NAME, null);
+ String clz = getStringAttribute(element, CLASS, null);
+ if (func == null || clz == null)
+ throw new DataImportHandlerException(
+ DataImportHandlerException.SEVERE,
+ "<function> must have a 'name' and 'class' attributes");
+ try {
+ evaluators.put(func, (Evaluator) DocBuilder.loadClass(clz)
+ .newInstance());
+ } catch (Exception exp) {
+ throw new DataImportHandlerException(
+ DataImportHandlerException.SEVERE,
+ "Unable to instantiate evaluator: " + clz, exp);
+ }
+ }
+ }
+ n = getChildNodes(e, DATA_SRC);
+ if (!n.isEmpty()) {
+ for (Element element : n) {
+ Properties p = new Properties();
+ HashMap<String, String> attrs = getAllAttributes(element);
+ for (Map.Entry<String, String> entry : attrs.entrySet()) {
+ p.setProperty(entry.getKey(), entry.getValue());
+ }
+ dataSources.put(p.getProperty("name"), p);
+ }
+ }
+ }
+
+ private static String getStringAttribute(Element e, String name, String def) {
+ String r = e.getAttribute(name);
+ if (r == null || "".equals(r.trim()))
+ r = def;
+ return r;
+ }
+
+ private static HashMap<String, String> getAllAttributes(Element e) {
+ HashMap<String, String> m = new HashMap<String, String>();
+ NamedNodeMap nnm = e.getAttributes();
+ for (int i = 0; i < nnm.getLength(); i++) {
+ m.put(nnm.item(i).getNodeName(), nnm.item(i).getNodeValue());
+ }
+ return m;
+ }
+
+ public static String getTxt(Node elem, StringBuffer buffer) {
+
+ if (elem.getNodeType() != Node.CDATA_SECTION_NODE) {
+ NodeList childs = elem.getChildNodes();
+ for (int i = 0; i < childs.getLength(); i++) {
+ Node child = childs.item(i);
+ short childType = child.getNodeType();
+ if (childType != Node.COMMENT_NODE
+ && childType != Node.PROCESSING_INSTRUCTION_NODE) {
+ getTxt(child, buffer);
+ }
+ }
+ } else {
+ buffer.append(elem.getNodeValue());
+ }
+
+ return buffer.toString();
+ }
+
+ public static List<Element> getChildNodes(Element e, String byName) {
+ List<Element> result = new ArrayList<Element>();
+ NodeList l = e.getChildNodes();
+ for (int i = 0; i < l.getLength(); i++) {
+ if (e.equals(l.item(i).getParentNode())
+ && byName.equals(l.item(i).getNodeName()))
+ result.add((Element) l.item(i));
+ }
+ return result;
+ }
+
+ public void clearCaches() {
+ for (Document document : documents)
+ for (Entity entity : document.entities)
+ entity.clearCache();
+
+ }
+
+ public static final String SCRIPT = "script";
+
+ public static final String NAME = "name";
+
+ public static final String SCRIPT_LANG = "scriptlanguage";
+
+ public static final String SCRIPT_NAME = "scriptname";
+
+ public static final String PROCESSOR = "processor";
+
+ public static final String IMPORTER_NS = "dataimporter";
+
+ public static final String ROOT_ENTITY = "rootEntity";
+
+ public static final String FUNCTION = "function";
+
+ public static final String CLASS = "class";
+
+ public static final String DATA_SRC = "dataSource";
+
+}
Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Added: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java?rev=681182&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java (added)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java Wed Jul 30 12:35:58 2008
@@ -0,0 +1,394 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.apache.lucene.document.Document;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStreamBase;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.SolrConfig;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.SolrResourceLoader;
+import org.apache.solr.handler.RequestHandlerBase;
+import org.apache.solr.handler.RequestHandlerUtils;
+import org.apache.solr.request.RawResponseWriter;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrQueryResponse;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.update.DocumentBuilder;
+import org.apache.solr.update.UpdateHandler;
+import org.apache.solr.util.plugin.SolrCoreAware;
+
+import java.util.*;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * <p>
+ * Solr Request Handler for data import from databases and REST data sources.
+ * </p>
+ * <p>
+ * It is configured in solrconfig.xml
+ * </p>
+ * <p/>
+ * <p>
+ * Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * </p>
+ * <p/>
+ * <b>This API is experimental and subject to change</b>
+ *
+ * @version $Id$
+ * @since solr 1.3
+ */
+public class DataImportHandler extends RequestHandlerBase implements
+ SolrCoreAware {
+
+ private static final Logger LOG = Logger.getLogger(DataImportHandler.class
+ .getName());
+
+ private DataImporter importer;
+
+ private Map<String, String> variables = new HashMap<String, String>();
+
+ @SuppressWarnings("unchecked")
+ private NamedList initArgs;
+
+ private Map<String, Properties> dataSources = new HashMap<String, Properties>();
+
+ private DataImporter.RequestParams requestParams;
+
+ private List<Document> debugDocuments;
+
+ private DebugLogger debugLogger;
+
+ private boolean debugEnabled = true;
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void init(NamedList args) {
+ super.init(args);
+
+ initArgs = args;
+ }
+
+ @SuppressWarnings("unchecked")
+ public void inform(SolrCore core) {
+ try {
+ String debug = (String) initArgs.get(ENABLE_DEBUG);
+ if (debug != null && "no".equals(debug))
+ debugEnabled = false;
+ NamedList defaults = (NamedList) initArgs.get("defaults");
+ if (defaults != null) {
+ String configLoc = (String) defaults.get("config");
+ if (configLoc != null && configLoc.length() != 0) {
+ processConfiguration(defaults);
+
+ importer = new DataImporter(SolrWriter.getResourceAsString(core
+ .getResourceLoader().openResource(configLoc)), core,
+ dataSources);
+ }
+ }
+ } catch (Throwable e) {
+ SolrConfig.severeErrors.add(e);
+ LOG.log(Level.SEVERE, DataImporter.MSG.LOAD_EXP, e);
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ DataImporter.MSG.INVALID_CONFIG, e);
+ }
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp)
+ throws Exception {
+ rsp.setHttpCaching(false);
+ SolrParams params = req.getParams();
+ requestParams = new DataImporter.RequestParams(getParamsMap(params));
+ String command = requestParams.command;
+
+ if (DataImporter.SHOW_CONF_CMD.equals(command)) {
+ // Modify incoming request params to add wt=raw
+ ModifiableSolrParams rawParams = new ModifiableSolrParams(req.getParams());
+ rawParams.set(CommonParams.WT, "raw");
+ req.setParams(rawParams);
+ String dataConfigFile = defaults.get("config");
+ ContentStreamBase content = new ContentStreamBase.StringStream(SolrWriter
+ .getResourceAsString(req.getCore().getResourceLoader().openResource(
+ dataConfigFile)));
+ rsp.add(RawResponseWriter.CONTENT, content);
+ return;
+ }
+
+ rsp.add("initArgs", initArgs);
+ String message = "";
+
+ if (command != null)
+ rsp.add("command", command);
+
+ if (requestParams.debug) {
+ // Reload the data-config.xml
+ importer = null;
+ if (requestParams.dataConfig != null) {
+ try {
+ processConfiguration((NamedList) initArgs.get("defaults"));
+ importer = new DataImporter(requestParams.dataConfig, req.getCore()
+ , dataSources);
+ } catch (RuntimeException e) {
+ rsp.add("exception", DebugLogger.getStacktraceString(e));
+ importer = null;
+ return;
+ }
+ } else {
+ inform(req.getCore());
+ }
+ message = DataImporter.MSG.CONFIG_RELOADED;
+ }
+
+ // If importer is still null
+ if (importer == null) {
+ rsp.add("status", DataImporter.MSG.NO_INIT);
+ return;
+ }
+
+ if (command != null && DataImporter.ABORT_CMD.equals(command)) {
+ importer.rumCmd(requestParams, null, null);
+ } else if (importer.getStatus() != DataImporter.Status.IDLE) {
+ message = DataImporter.MSG.CMD_RUNNING;
+ } else if (command != null) {
+ if (DataImporter.FULL_IMPORT_CMD.equals(command)
+ || DataImporter.DELTA_IMPORT_CMD.equals(command)) {
+ UpdateHandler updater = req.getCore().getUpdateHandler();
+ SolrResourceLoader loader = req.getCore().getResourceLoader();
+ SolrWriter sw = getSolrWriter(updater, loader, req
+ .getSchema());
+
+ if (requestParams.debug) {
+ if (debugEnabled) {
+ // Synchronous request for the debug mode
+ importer.rumCmd(requestParams, sw, variables);
+ rsp.add("mode", "debug");
+ rsp.add("documents", debugDocuments);
+ if (debugLogger != null)
+ rsp.add("verbose-output", debugLogger.output);
+ debugLogger = null;
+ debugDocuments = null;
+ } else {
+ message = DataImporter.MSG.DEBUG_NOT_ENABLED;
+ }
+ } else {
+ // Asynchronous request for normal mode
+ importer.runAsync(requestParams, sw, variables);
+ }
+ } else if (DataImporter.RELOAD_CONF_CMD.equals(command)) {
+ importer = null;
+ inform(req.getCore());
+ message = DataImporter.MSG.CONFIG_RELOADED;
+ }
+ }
+ rsp.add("status", importer.getStatus() == DataImporter.Status.IDLE ? "idle"
+ : "busy");
+ rsp.add("importResponse", message);
+ rsp.add("statusMessages", importer.getStatusMessages());
+
+ RequestHandlerUtils.addExperimentalFormatWarning(rsp);
+ }
+
+ private Map<String, Object> getParamsMap(SolrParams params) {
+ Iterator<String> names = params.getParameterNamesIterator();
+ Map<String, Object> result = new HashMap<String, Object>();
+ while (names.hasNext()) {
+ String s = names.next();
+ String[] val = params.getParams(s);
+ if (val == null || val.length < 1)
+ continue;
+ if (val.length == 1)
+ result.put(s, val[0]);
+ else
+ result.put(s, Arrays.asList(val));
+ }
+ return result;
+ }
+
+ @SuppressWarnings("unchecked")
+ private void processConfiguration(NamedList defaults) {
+ if (defaults == null) {
+ LOG
+ .info("No configuration specified in solrconfig.xml for DataImportHandler");
+ return;
+ }
+
+ LOG.info("Processing configuration from solrconfig.xml: " + defaults);
+
+ dataSources = new HashMap<String, Properties>();
+ variables = new HashMap<String, String>();
+
+ int position = 0;
+
+ while (position < defaults.size()) {
+ if (defaults.getName(position) == null)
+ break;
+
+ String name = defaults.getName(position);
+ if (name.equals("datasource")) {
+ NamedList dsConfig = (NamedList) defaults.getVal(position);
+ Properties props = new Properties();
+ for (int i = 0; i < dsConfig.size(); i++)
+ props.put(dsConfig.getName(i), dsConfig.getVal(i));
+ LOG.info("Adding properties to datasource: " + props);
+ dataSources.put((String) dsConfig.get("name"), props);
+ } else if (!name.equals("config")) {
+ String value = (String) defaults.getVal(position);
+ variables.put(name, value);
+ }
+ position++;
+ }
+ }
+
+ private SolrWriter getSolrWriter(final UpdateHandler updater,
+ final SolrResourceLoader loader, final IndexSchema schema) {
+
+ return new SolrWriter(updater, loader.getConfigDir()) {
+
+ @Override
+ public boolean upload(SolrDoc d) {
+ try {
+ Document document = DocumentBuilder.toDocument(
+ ((SolrDocumentWrapper) d).doc, schema);
+ if (requestParams.debug) {
+ if (debugDocuments == null)
+ debugDocuments = new ArrayList<Document>();
+ debugDocuments.add(document);
+ if (debugDocuments.size() >= requestParams.rows) {
+ // Abort this operation now
+ importer.getDocBuilder().abort();
+ }
+ }
+
+ return super.upload(document);
+ } catch (RuntimeException e) {
+ LOG.log(Level.SEVERE, "Exception while adding: " + d, e);
+ return false;
+ }
+ }
+
+ public void log(int event, String name, Object row) {
+ if (debugLogger == null) {
+ debugLogger = new DebugLogger();
+ }
+ debugLogger.log(event, name, row);
+ }
+
+
+ public Class loadClass(String name) throws ClassNotFoundException {
+ return loader.findClass(name);
+ }
+
+ public SolrDoc getSolrDocInstance() {
+ return new SolrDocumentWrapper();
+ }
+ };
+ }
+
+ static class SolrDocumentWrapper implements SolrWriter.SolrDoc {
+ SolrInputDocument doc;
+
+ public SolrDocumentWrapper() {
+ doc = new SolrInputDocument();
+ }
+
+ public void setDocumentBoost(float boost) {
+ doc.setDocumentBoost(boost);
+ }
+
+ public Object getField(String field) {
+ return doc.getField(field);
+ }
+
+ public void addField(String name, Object value, float boost) {
+ doc.addField(name, value, boost);
+ }
+
+ public String toString() {
+ return doc.toString();
+ }
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public NamedList getStatistics() {
+ if (importer == null)
+ return super.getStatistics();
+
+ DocBuilder.Statistics cumulative = importer.cumulativeStatistics;
+ NamedList result = new NamedList();
+
+ result.add("Status", importer.getStatus().toString());
+
+ if (importer.docBuilder != null) {
+ DocBuilder.Statistics running = importer.docBuilder.importStatistics;
+ result.add("Documents Processed", running.docCount);
+ result.add("Requests made to DataSource", running.queryCount);
+ result.add("Rows Fetched", running.rowsCount);
+ result.add("Documents Deleted", running.deletedDocCount);
+ result.add("Documents Skipped", running.skipDocCount);
+ }
+
+ result.add(DataImporter.MSG.TOTAL_DOC_PROCESSED, cumulative.docCount);
+ result.add(DataImporter.MSG.TOTAL_QUERIES_EXECUTED, cumulative.queryCount);
+ result.add(DataImporter.MSG.TOTAL_ROWS_EXECUTED, cumulative.rowsCount);
+ result.add(DataImporter.MSG.TOTAL_DOCS_DELETED, cumulative.deletedDocCount);
+ result.add(DataImporter.MSG.TOTAL_DOCS_SKIPPED, cumulative.skipDocCount);
+
+ NamedList requestStatistics = super.getStatistics();
+ if (requestStatistics != null) {
+ for (int i = 0; i < requestStatistics.size(); i++) {
+ result.add(requestStatistics.getName(i), requestStatistics.getVal(i));
+ }
+ }
+
+ return result;
+ }
+
+ // //////////////////////SolrInfoMBeans methods //////////////////////
+
+ @Override
+ public String getDescription() {
+ return DataImporter.MSG.JMX_DESC;
+ }
+
+ @Override
+ public String getSourceId() {
+ return "$Id$";
+ }
+
+ @Override
+ public String getVersion() {
+ return "1.0";
+ }
+
+ @Override
+ public String getSource() {
+ return "$URL$";
+ }
+
+ public static final String ENABLE_DEBUG = "enableDebug";
+}
Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Added: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java?rev=681182&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java (added)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java Wed Jul 30 12:35:58 2008
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.dataimport;
+
+/**
+ * <p>
+ * Exception class for all DataImportHandler exceptions
+ * </p>
+ * <p/>
+ * <b>This API is experimental and subject to change</b>
+ * <p/>
+ * $Id$
+ *
+ * @since solr 1.3
+ */
+public class DataImportHandlerException extends RuntimeException {
+ private int errCode;
+
+ public boolean debugged = false;
+
+ public static final int SEVERE = 500, WARN = 400, SKIP = 300;
+
+ public DataImportHandlerException(int err) {
+ super();
+ errCode = err;
+ }
+
+ public DataImportHandlerException(int err, String message) {
+ super(message + MSG + SolrWriter.getDocCount());
+ errCode = err;
+ }
+
+ public DataImportHandlerException(int err, String message, Throwable cause) {
+ super(message + MSG + SolrWriter.getDocCount(), cause);
+ errCode = err;
+ }
+
+ public DataImportHandlerException(int err, Throwable cause) {
+ super(cause);
+ errCode = err;
+ }
+
+ public int getErrCode() {
+ return errCode;
+ }
+
+ public static final String MSG = " Processing Documemt # ";
+}
Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Added: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java?rev=681182&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java (added)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java Wed Jul 30 12:35:58 2008
@@ -0,0 +1,544 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.dataimport;
+
+import org.apache.solr.core.SolrConfig;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaField;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.xml.sax.InputSource;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import java.io.StringReader;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * <p>
+ * Stores all configuration information for pulling and indexing data.
+ * </p>
+ * <p/>
+ * <b>This API is experimental and subject to change</b>
+ *
+ * @version $Id$
+ * @since solr 1.3
+ */
+public class DataImporter {
+
+ public enum Status {
+ IDLE, RUNNING_FULL_DUMP, RUNNING_DELTA_DUMP, JOB_FAILED
+ }
+
+ private static final Logger LOG = Logger.getLogger(DataImporter.class
+ .getName());
+
+ private Status status = Status.IDLE;
+
+ private DataConfig config;
+
+ private Date lastIndexTime;
+
+ private Date indexStartTime;
+
+ private Properties store = new Properties();
+
+ private Map<String, Properties> dataSourceProps;
+
+ private IndexSchema schema;
+
+ public DocBuilder docBuilder;
+
+ public DocBuilder.Statistics cumulativeStatistics = new DocBuilder.Statistics();
+
+ public Map<String, Evaluator> evaluators;
+
+ private SolrCore core;
+
+ /**
+ * Only for testing purposes
+ */
+ DataImporter() {
+ }
+
+ public DataImporter(String dataConfig, SolrCore core,
+ Map<String, Properties> ds) {
+ if (dataConfig == null)
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+ "Configuration not found");
+ this.core = core;
+ this.schema = core.getSchema();
+ dataSourceProps = ds;
+ loadDataConfig(dataConfig);
+
+ for (DataConfig.Document document : config.documents) {
+ for (DataConfig.Entity e : document.entities) {
+ Map<String, DataConfig.Field> fields = new HashMap<String, DataConfig.Field>();
+ initEntity(e, fields, false);
+ e.implicitFields = new ArrayList<DataConfig.Field>();
+ String errs = verifyWithSchema(fields, e.implicitFields);
+ if (e.implicitFields.isEmpty())
+ e.implicitFields = null;
+ if (errs != null) {
+ throw new DataImportHandlerException(
+ DataImportHandlerException.SEVERE, errs);
+ }
+ }
+ }
+ }
+
+ private String verifyWithSchema(Map<String, DataConfig.Field> fields,
+ List<DataConfig.Field> autoFields) {
+ List<String> errors = new ArrayList<String>();
+ Map<String, SchemaField> schemaFields = schema.getFields();
+ for (Map.Entry<String, SchemaField> entry : schemaFields.entrySet()) {
+ SchemaField sf = entry.getValue();
+ if (!fields.containsKey(sf.getName())) {
+ if (sf.isRequired()) {
+ LOG
+ .info(sf.getName()
+ + " is a required field in SolrSchema . But not found in DataConfig");
+ }
+ autoFields.add(new DataConfig.Field(sf.getName(), sf.multiValued()));
+ }
+ }
+ for (Map.Entry<String, DataConfig.Field> entry : fields.entrySet()) {
+ DataConfig.Field fld = entry.getValue();
+ FieldType fieldType = null;
+
+ try {
+ fieldType = schema.getDynamicFieldType(fld.name);
+ } catch (RuntimeException e) {
+ // Ignore because it may not be a dynamic field
+ }
+
+ if (fld.name != null) {
+ if (schema.getFields().get(fld.name) == null && fieldType == null) {
+ errors
+ .add("The field :"
+ + fld.name
+ + " present in DataConfig does not have a counterpart in Solr Schema");
+ }
+ } else if (schema.getFields().get(fld.column) == null
+ && fieldType == null) {
+ LOG.info("Column : " + fld.column + " is not a schema field");
+ }
+ }
+
+ if (!errors.isEmpty()) {
+ StringBuffer sb = new StringBuffer("There are errors in the Schema\n");
+ for (String error : errors) {
+ sb.append(error).append("\n");
+ }
+ return sb.toString();
+
+ }
+ return null;
+
+ }
+
+ void loadDataConfig(String configFile) {
+
+ try {
+ DocumentBuilder builder = DocumentBuilderFactory.newInstance()
+ .newDocumentBuilder();
+ Document document = builder.parse(new InputSource(new StringReader(
+ configFile)));
+
+ config = new DataConfig();
+ config.readFromXml((Element) document.getElementsByTagName("dataConfig")
+ .item(0));
+
+ LOG.info("Data Configuration loaded successfully");
+ } catch (Exception e) {
+ SolrConfig.severeErrors.add(e);
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+ "Exception occurred while initializing context", e);
+ }
+ }
+
+ private void initEntity(DataConfig.Entity e,
+ Map<String, DataConfig.Field> fields, boolean docRootFound) {
+ if (e.pk != null)
+ e.primaryKeys = e.pk.split(",");
+ e.allAttributes.put(DATA_SRC, e.dataSource);
+
+ if (!docRootFound && !"false".equals(e.docRoot)) {
+ // if in this chain no document root is found()
+ e.isDocRoot = true;
+ }
+
+ if (e.fields != null) {
+ for (DataConfig.Field f : e.fields) {
+ f.nameOrColName = f.getName();
+ SchemaField schemaField = schema.getFields().get(f.getName());
+ if (schemaField != null) {
+ f.multiValued = schemaField.multiValued();
+ f.allAttributes.put(MULTI_VALUED, Boolean.toString(schemaField
+ .multiValued()));
+ f.allAttributes.put(TYPE, schemaField.getType().getTypeName());
+ f.allAttributes.put("indexed", Boolean
+ .toString(schemaField.indexed()));
+ f.allAttributes.put("stored", Boolean.toString(schemaField.stored()));
+ f.allAttributes.put("defaultValue", schemaField.getDefaultValue());
+ } else {
+
+ try {
+ f.allAttributes.put(TYPE, schema.getDynamicFieldType(f.getName())
+ .getTypeName());
+ f.allAttributes.put(MULTI_VALUED, "true");
+ f.multiValued = true;
+ } catch (RuntimeException e2) {
+ LOG.info("Field in data-config.xml - " + f.getName()
+ + " not found in schema.xml");
+ f.toWrite = false;
+ }
+ }
+ fields.put(f.getName(), f);
+ f.entity = e;
+ f.allAttributes.put("boost", f.boost.toString());
+ f.allAttributes.put("toWrite", Boolean.toString(f.toWrite));
+ e.allFieldsList.add(Collections.unmodifiableMap(f.allAttributes));
+ }
+ }
+ e.allFieldsList = Collections.unmodifiableList(e.allFieldsList);
+ e.allAttributes = Collections.unmodifiableMap(e.allAttributes);
+
+ addDataSource(e);
+
+ if (e.entities == null)
+ return;
+ for (DataConfig.Entity e1 : e.entities) {
+ e1.parentEntity = e;
+ initEntity(e1, fields, e.isDocRoot || docRootFound);
+ }
+
+ }
+
+ public DataConfig getConfig() {
+ return config;
+ }
+
+ public Date getIndexStartTime() {
+ return indexStartTime;
+ }
+
+ public void setIndexStartTime(Date indextStartTime) {
+ this.indexStartTime = indextStartTime;
+ }
+
+ public Date getLastIndexTime() {
+ return lastIndexTime;
+ }
+
+ public void setLastIndexTime(Date lastIndexTime) {
+ this.lastIndexTime = lastIndexTime;
+ }
+
+ public void store(Object key, Object value) {
+ store.put(key, value);
+ }
+
+ public Object retrieve(Object key) {
+ return store.get(key);
+ }
+
+ @SuppressWarnings("unchecked")
+ public void addDataSource(DataConfig.Entity key) {
+ if ("null".equals(key.dataSource)) {
+ key.dataSrc = new MockDataSource();
+ return;
+ }
+ key.dataSrc = getDataSourceInstance(key);
+ }
+
+ DataSource getDataSourceInstance(DataConfig.Entity key) {
+ Properties p = dataSourceProps.get(key.dataSource);
+ if (p == null)
+ p = config.dataSources.get(key.dataSource);
+ if (p == null)
+ p = dataSourceProps.get(null);// for default data source
+ if (p == null)
+ p = config.dataSources.get(null);
+ if (p == null)
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+ "No dataSource :" + key.dataSource + " available for entity :"
+ + key.name);
+ String impl = p.getProperty(TYPE);
+ DataSource dataSrc = null;
+ if (impl == null) {
+ dataSrc = new JdbcDataSource();
+ } else {
+ try {
+ dataSrc = (DataSource) DocBuilder.loadClass(impl).newInstance();
+ } catch (Exception e) {
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+ "Invalid type for data source: " + impl, e);
+ }
+ }
+ try {
+ Properties copyProps = new Properties();
+ copyProps.putAll(p);
+ dataSrc.init(new ContextImpl(key, null, dataSrc, 0,
+ Collections.EMPTY_MAP, new HashMap(), null, this), copyProps);
+ } catch (Exception e) {
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+ "Failed to initialize DataSource: " + key.dataSource, e);
+ }
+ return dataSrc;
+ }
+
+ public Status getStatus() {
+ return status;
+ }
+
+ public void setStatus(Status status) {
+ this.status = status;
+ }
+
+ public void doFullImport(SolrWriter writer, RequestParams requestParams,
+ Map<String, String> variables) {
+ LOG.info("Starting Full Import");
+ setStatus(Status.RUNNING_FULL_DUMP);
+
+ if (requestParams.commit)
+ setIndexStartTime(new Date());
+
+ try {
+ if (requestParams.clean)
+ writer.doDeleteAll();
+ docBuilder = new DocBuilder(this, writer, requestParams, variables);
+ docBuilder.execute(getConfig().documents.get(0).name);
+ if (!requestParams.debug)
+ cumulativeStatistics.add(docBuilder.importStatistics);
+ } catch (RuntimeException e) {
+ LOG.log(Level.SEVERE, "Full Import failed", e);
+ } finally {
+ setStatus(Status.IDLE);
+ config.clearCaches();
+ DocBuilder.INSTANCE.set(null);
+ }
+
+ }
+
+ public void doDeltaImport(SolrWriter writer, RequestParams requestParams,
+ Map<String, String> variables) {
+ LOG.info("Starting Delta Import");
+ setStatus(Status.RUNNING_DELTA_DUMP);
+
+ try {
+ if (requestParams.commit) {
+ Date lastModified = writer.loadIndexStartTime();
+ setIndexStartTime(new Date());
+ setLastIndexTime(lastModified);
+ }
+ docBuilder = new DocBuilder(this, writer, requestParams, variables);
+ docBuilder.execute(config.documents.get(0).name);
+ if (!requestParams.debug)
+ cumulativeStatistics.add(docBuilder.importStatistics);
+ } catch (RuntimeException e) {
+ LOG.log(Level.SEVERE, "Delta Import Failed", e);
+ } finally {
+ setStatus(Status.IDLE);
+ config.clearCaches();
+ DocBuilder.INSTANCE.set(null);
+ }
+
+ }
+
+ public void runAsync(final RequestParams reqParams, final SolrWriter sw,
+ final Map<String, String> variables) {
+ new Thread() {
+ @Override
+ public void run() {
+ rumCmd(reqParams, sw, variables);
+ }
+ }.start();
+ }
+
+ void rumCmd(RequestParams reqParams, SolrWriter sw,
+ Map<String, String> variables) {
+ String command = reqParams.command;
+ if (command.equals("full-import")) {
+ doFullImport(sw, reqParams, variables);
+ } else if (command.equals(DELTA_IMPORT_CMD)) {
+ doDeltaImport(sw, reqParams, variables);
+ } else if (command.equals(ABORT_CMD)) {
+ if (docBuilder != null)
+ docBuilder.abort();
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ Map<String, String> getStatusMessages() {
+ Map statusMessages = (Map) retrieve(STATUS_MSGS);
+ Map<String, String> result = new LinkedHashMap<String, String>();
+ if (statusMessages != null) {
+ for (Object o : statusMessages.entrySet()) {
+ Map.Entry e = (Map.Entry) o;
+ result.put((String) e.getKey(), e.getValue().toString());
+ }
+ }
+ return result;
+
+ }
+
+ public DocBuilder getDocBuilder() {
+ return docBuilder;
+ }
+
+ public static final ThreadLocal<AtomicLong> QUERY_COUNT = new ThreadLocal<AtomicLong>() {
+ protected AtomicLong initialValue() {
+ return new AtomicLong();
+ }
+ };
+
+ static final SimpleDateFormat DATE_TIME_FORMAT = new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss");
+
+ static final class MSG {
+ public static final String NO_CONFIG_FOUND = "Configuration not found";
+
+ public static final String NO_INIT = "DataImportHandler started. Not Initialized. No commands can be run";
+
+ public static final String INVALID_CONFIG = "FATAL: Could not create importer. DataImporter config invalid";
+
+ public static final String LOAD_EXP = "Exception while loading DataImporter";
+
+ public static final String JMX_DESC = "Manage data import from databases to Solr";
+
+ public static final String CMD_RUNNING = "A command is still running...";
+
+ public static final String DEBUG_NOT_ENABLED = "Debug not enabled. Add a tag <str name=\"enableDebug\">true</str> in solrconfig.xml";
+
+ public static final String CONFIG_RELOADED = "Configuration Re-loaded sucessfully";
+
+ public static final String TOTAL_DOC_PROCESSED = "Total Documents Processed";
+
+ public static final String TOTAL_QUERIES_EXECUTED = "Total Requests made to DataSource";
+
+ public static final String TOTAL_ROWS_EXECUTED = "Total Rows Fetched";
+
+ public static final String TOTAL_DOCS_DELETED = "Total Documents Deleted";
+
+ public static final String TOTAL_DOCS_SKIPPED = "Total Documents Skipped";
+ }
+
+ static final class RequestParams {
+ public String command = null;
+
+ public boolean debug = false;
+
+ public boolean verbose = false;
+
+ public boolean commit = true;
+
+ public boolean optimize = true;
+
+ public int start = 0;
+
+ public int rows = 10;
+
+ public boolean clean = true;
+
+ public List<String> entities;
+
+ public Map<String, Object> requestParams;
+
+ public String dataConfig;
+
+ public RequestParams() {
+ }
+
+ public RequestParams(Map<String, Object> requestParams) {
+ if (requestParams.containsKey("command"))
+ command = (String) requestParams.get("command");
+
+ if ("on".equals(requestParams.get("debug"))) {
+ debug = true;
+ // Set default values suitable for debug mode
+ commit = false;
+ clean = false;
+ verbose = "true".equals(requestParams.get("verbose"))
+ || "on".equals(requestParams.get("verbose"));
+ }
+ if (requestParams.containsKey("commit"))
+ commit = Boolean.parseBoolean((String) requestParams.get("commit"));
+ if (requestParams.containsKey("start"))
+ start = Integer.parseInt((String) requestParams.get("start"));
+ if (requestParams.containsKey("rows"))
+ rows = Integer.parseInt((String) requestParams.get("rows"));
+ if (requestParams.containsKey("clean"))
+ clean = Boolean.parseBoolean((String) requestParams.get("clean"));
+ if (requestParams.containsKey("optimize"))
+ optimize = Boolean.parseBoolean((String) requestParams.get("optimize"));
+
+ Object o = requestParams.get("entity");
+
+ if (o != null && o instanceof String) {
+ entities = new ArrayList<String>();
+ entities.add((String) o);
+ } else if (o != null && o instanceof List) {
+ entities = (List<String>) requestParams.get("entity");
+ }
+
+ dataConfig = (String) requestParams.get("dataConfig");
+ if (dataConfig != null && dataConfig.trim().length() == 0) {
+ // Empty data-config param is not valid, change it to null
+ dataConfig = null;
+ }
+
+ this.requestParams = requestParams;
+ }
+ }
+
+
+ public SolrCore getCore() {
+ return core;
+ }
+
+ public static final String COLUMN = "column";
+
+ public static final String TYPE = "type";
+
+ public static final String DATA_SRC = "dataSource";
+
+ public static final String MULTI_VALUED = "multiValued";
+
+ public static final String NAME = "name";
+
+ public static final String STATUS_MSGS = "status-messages";
+
+ public static final String FULL_IMPORT_CMD = "full-import";
+
+ public static final String DELTA_IMPORT_CMD = "delta-import";
+
+ public static final String ABORT_CMD = "abort";
+
+ public static final String DEBUG_MODE = "debug";
+
+ public static final String RELOAD_CONF_CMD = "reload-config";
+
+ public static final String SHOW_CONF_CMD = "show-config";
+}
Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Added: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataSource.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataSource.java?rev=681182&view=auto
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataSource.java (added)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataSource.java Wed Jul 30 12:35:58 2008
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.dataimport;
+
+import java.util.Properties;
+
+/**
+ * <p>
+ * Provides data from a source with a given query.
+ * </p>
+ * <p/>
+ * <p>
+ * Implementation of this interface must provide a default no-arg constructor
+ * </p>
+ * <p/>
+ * <p>
+ * Refer to <a
+ * href="http://wiki.apache.org/solr/DataImportHandler">http://wiki.apache.org/solr/DataImportHandler</a>
+ * for more details.
+ * </p>
+ * <p/>
+ * <b>This API is experimental and may change in the future.</b>
+ *
+ * @version $Id$
+ * @since solr 1.3
+ */
+public abstract class DataSource<T> {
+
+ /**
+ * Initializes the DataSource with the <code>Context</code> and
+ * initialization properties.
+ * <p/>
+ * This is invoked by the <code>DataImporter</code> after creating an
+ * instance of this class.
+ *
+ * @param context
+ * @param initProps
+ */
+ public abstract void init(Context context, Properties initProps);
+
+ /**
+ * Get records for the given query.The return type depends on the
+ * implementation .
+ *
+ * @param query The query string. It can be a SQL for JdbcDataSource or a URL
+ * for HttpDataSource or a file location for FileDataSource or a custom
+ * format for your own custom DataSource.
+ * @return Depends on the implementation. For instance JdbcDataSource returns
+ * an Iterator<Map <String,Object>>
+ */
+ public abstract T getData(String query);
+
+ /**
+ * Cleans up resources of this DataSource after use.
+ */
+ public abstract void close();
+}
Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataSource.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataSource.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL