You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2012/07/19 17:59:32 UTC
svn commit: r1363400 [23/31] - in /lucene/dev/branches/pforcodec_3892: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/
dev-tools/idea/.idea/copyright/ dev-tools/idea/.idea/libraries/
dev-tools/idea/lucene/ dev-tools/maven/ dev-tools/maven/lucene...
Modified: lucene/dev/branches/pforcodec_3892/lucene/tools/src/java/lucene-solr.antlib.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/tools/src/java/lucene-solr.antlib.xml?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/tools/src/java/lucene-solr.antlib.xml (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/tools/src/java/lucene-solr.antlib.xml Thu Jul 19 15:58:54 2012
@@ -18,4 +18,7 @@
<taskdef
name="licenses"
classname="org.apache.lucene.validation.LicenseCheckTask" />
+ <taskdef
+ name="forbidden-apis"
+ classname="org.apache.lucene.validation.ForbiddenApisCheckTask" />
</antlib>
Modified: lucene/dev/branches/pforcodec_3892/lucene/tools/src/java/org/apache/lucene/validation/LicenseCheckTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/tools/src/java/org/apache/lucene/validation/LicenseCheckTask.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/tools/src/java/org/apache/lucene/validation/LicenseCheckTask.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/tools/src/java/org/apache/lucene/validation/LicenseCheckTask.java Thu Jul 19 15:58:54 2012
@@ -140,7 +140,7 @@ public class LicenseCheckTask extends Ta
checked++;
}
- log(String.format(Locale.ENGLISH,
+ log(String.format(Locale.ROOT,
"Scanned %d JAR file(s) for licenses (in %.2fs.), %d error(s).",
checked, (System.currentTimeMillis() - start) / 1000.0, errors),
errors > 0 ? Project.MSG_ERR : Project.MSG_INFO);
@@ -255,7 +255,7 @@ outer:
private static final String createChecksumString(byte[] digest) {
StringBuilder checksum = new StringBuilder();
for (int i = 0; i < digest.length; i++) {
- checksum.append(String.format(Locale.ENGLISH, "%02x",
+ checksum.append(String.format(Locale.ROOT, "%02x",
CHECKSUM_BYTE_MASK & digest[i]));
}
return checksum.toString();
Modified: lucene/dev/branches/pforcodec_3892/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/CHANGES.txt?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/CHANGES.txt (original)
+++ lucene/dev/branches/pforcodec_3892/solr/CHANGES.txt Thu Jul 19 15:58:54 2012
@@ -24,6 +24,124 @@ $Id$
(No changes)
+================== 4.0.0-BETA ===================
+
+Versions of Major Components
+---------------------
+Apache Tika 1.1
+Carrot2 3.5.0
+Velocity 1.6.4 and Velocity Tools 2.0
+Apache UIMA 2.3.1
+Apache ZooKeeper 3.3.5
+
+Detailed Change List
+----------------------
+
+New Features
+----------------------
+
+* LUCENE-4201: Added JapaneseIterationMarkCharFilterFactory to normalize Japanese
+ iteration marks. (Robert Muir, Christian Moen)
+
+* SOLR-1856: In Solr Cell, literals should override Tika-parsed values.
+ Patch adds a param "literalsOverride" which defaults to true, but can be set
+ to "false" to let Tika-parsed values be appended to literal values (Chris Harris, janhoy)
+
+* SOLR-3488: Added a Collection management API for SolrCloud.
+ (Tommaso Teofili, Sami Siren, yonik, Mark Miller)
+
+* SOLR-3559: Full deleteByQuery support with SolrCloud distributed indexing.
+ All replicas of a shard will be consistent, even if updates arrive in a
+ different order on different replicas. (yonik)
+
+* SOLR-1929: Index encrypted documents with ExtractingUpdateRequestHandler.
+ By supplying resource.password=<mypw> or specifying an external file with regular
+ expressions matching file names, Solr will decrypt and index PDFs and DOCX formats.
+ (janhoy, Yiannis Pericleous)
+
+* SOLR-3562: Add options to remove instance dir or data dir on core unload.
+ (Mark Miller, Per Steffensen)
+
+* SOLR-2702: The default directory factory was changed to NRTCachingDirectoryFactory
+ which wraps the StandardDirectoryFactory and caches small files for improved
+ Near Real-time (NRT) performance. (Mark Miller, yonik)
+
+* SOLR-2616: Include a sample java util logging configuration file.
+ (David Smiley, Mark Miller)
+
+* SOLR-3460: Add cloud-scripts directory and a zkcli.sh|bat tool for easy scripting
+ and interaction with ZooKeeper. (Mark Miller)
+
+* SOLR-1725: StatelessScriptUpdateProcessorFactory allows users to implement
+ the full ScriptUpdateProcessor API using any scripting language with a
+ javax.script.ScriptEngineFactory
+ (Uri Boness, ehatcher, Simon Rosenthal, hossman)
+
+* SOLR-139: Change to updateable documents to create the document if it doesn't
+ already exist. To assert that the document must exist, use the optimistic
+ concurrency feature by specifying a _version_ of 1. (yonik)
+
+
+Bug Fixes
+----------------------
+
+* SOLR-3582: Our ZooKeeper watchers respond to session events as if they are change events,
+ creating undesirable side effects. (Trym R. Møller, Mark Miller)
+
+* SOLR-3467: ExtendedDismax escaping is missing several reserved characters
+ (Michael Dodsworth via janhoy)
+
+* SOLR-3587: After reloading a SolrCore, the original Analyzer is still used rather than a new
+ one. (Alexey Serba, yonik, rmuir, Mark Miller)
+
+* LUCENE-4185: Fix a bug where CharFilters were wrongly being applied twice. (Michael Froh, rmuir)
+
+* SOLR-3610: After reloading a core, indexing would fail on any newly added fields to the schema. (Brent Mills, rmuir)
+
+* SOLR-3377: edismax fails to correctly parse a fielded query wrapped by parens.
+ This regression was introduced in 3.6. (Bernd Fehling, Jan Høydahl, yonik)
+
+* SOLR-3621: Fix rare concurrency issue when opening a new IndexWriter for replication or rollback.
+ (Mark Miller)
+
+* SOLR-1781: Replication index directories not always cleaned up. (Terje Sten Bjerkseth, Mark Miller)
+
+* SOLR-3639: Update ZooKeeper to 3.3.5 for a variety of bug fixes. (Mark Miller)
+
+* SOLR-3629: Typo in solr.xml persistence when overriding the solrconfig.xml
+ file name using the "config" attribute prevented the override file from being
+ used. (Ryan Zezeski, hossman)
+
+
+Other Changes
+----------------------
+
+* SOLR-3524: Make discarding punctuation configurable in JapaneseTokenizerFactory.
+ The default is to discard punctuation, but this is overridable as an expert option.
+ (Kazuaki Hiraga, Jun Ohtani via Christian Moen)
+
+* SOLR-1770: Move the default core instance directory into a collection1 folder.
+ (Mark Miller)
+
+* SOLR-3355: Add shard and collection to SolrCore statistics. (Michael Garski, Mark Miller)
+
+* SOLR-3575: solr.xml should default to persist=true (Mark Miller)
+
+* SOLR-3563: Unloading all cores in a SolrCloud collection will now cause the removal of
+ that collection's meta data from ZooKeeper. (Mark Miller, Per Steffensen)
+
+* SOLR-3599: Add zkClientTimeout to solr.xml so that it's obvious how to change it and so
+ that you can change it with a system property. (Mark Miller)
+
+* SOLR-3609: Change Solr's expanded webapp directory to be at a consistent path called
+ solr-webapp rather than a temporary directory. (Mark Miller)
+
+* SOLR-3600: Raise the default zkClientTimeout from 10 seconds to 15 seconds. (Mark Miller)
+
+* SOLR-3215: Clone SolrInputDocument when distrib indexing so that update processors after
+ the distrib update process do not process the document twice. (Mark Miller)
+
+
================== 4.0.0-ALPHA ==================
More information about this release, including any errata related to the
release notes, upgrade instructions, or other changes may be found online at:
@@ -91,7 +209,7 @@ Upgrading from Solr 3.6-dev
* Due to low level changes to support SolrCloud, the uniqueKey field can no
longer be populated via <copyField/> or <field default=...> in the
- schema.xml. Users wishing to have Solr automaticly generate a uniqueKey
+ schema.xml. Users wishing to have Solr automatically generate a uniqueKey
value when adding documents should instead use an instance of
solr.UUIDUpdateProcessorFactory in their update processor chain. See
SOLR-2796 for more details.
@@ -366,6 +484,8 @@ New Features
to schema.xml's <copyField/> declaration but as an update processor that can
be combined with other processors in any order. (Jan Høydahl & hossman)
+* SOLR-3351: eDismax: ps2 and ps3 params (janhoy)
+
* SOLR-3542: Add WeightedFragListBuilder for FVH and set it to default fragListBuilder
in example solrconfig.xml. (Sebastian Lutze, koji)
@@ -668,7 +788,7 @@ Other Changes
* SOLR-3534: The Dismax and eDismax query parsers will fall back on the 'df' parameter
when 'qf' is absent. And if neither is present nor the schema default search field
then an exception will be thrown now. (dsmiley)
-
+
Documentation
----------------------
Modified: lucene/dev/branches/pforcodec_3892/solr/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/build.xml?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/build.xml (original)
+++ lucene/dev/branches/pforcodec_3892/solr/build.xml Thu Jul 19 15:58:54 2012
@@ -15,7 +15,7 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<project name="solr" default="usage">
+<project name="solr" default="usage" xmlns:ivy="antlib:org.apache.ivy.ant">
<description>Solr</description>
<target name="usage" description="Prints out instructions">
@@ -51,7 +51,7 @@
</manifest>
</jar>
<delete includeemptydirs="true">
- <fileset dir="${example}/work" includes="**/*"/>
+ <fileset dir="${example}/solr-webapp" includes="**/*"/>
</delete>
<echo>See ${example}/README.txt for how to run the Solr example configuration.</echo>
</target>
@@ -130,7 +130,7 @@
<target name="compile" description="Compile the source code."
depends="compile-core, compile-contrib"/>
<target name="test" description="Validate, then run core, solrj, and contrib unit tests."
- depends="validate, test-core, test-contrib"/>
+ depends="test-core, test-contrib"/>
<target name="test-core" description="Runs the core and solrj unit tests."
depends="test-solr-core, test-solrj"/>
<target name="compile-test" description="Compile unit tests."
@@ -170,8 +170,10 @@
</ant>
</target>
- <!-- Validation (license/ notice checks). -->
- <target name="validate" depends="compile-tools,resolve" description="Validate legal stuff.">
+ <!-- Validation (license/notice/api checks). -->
+ <target name="validate" depends="check-licenses,rat-sources,check-forbidden-apis" description="Validate stuff." />
+
+ <target name="check-licenses" depends="compile-tools,resolve,load-custom-tasks" description="Validate license stuff.">
<license-check-macro dir="${basedir}">
<additional-excludes>
<!-- Exclude start.jar only (it'd be weird to have a license file there?) -->
@@ -187,6 +189,39 @@
</additional-filters>
</license-check-macro>
</target>
+
+ <target name="check-forbidden-apis" depends="compile-tools,compile-test,load-custom-tasks,-check-forbidden-java-apis,-check-forbidden-test-apis"
+ description="Check forbidden API calls in compiled class files."/>
+
+ <target name="-check-forbidden-java-apis">
+ <forbidden-apis>
+ <classpath refid="additional.dependencies"/>
+ <apiFileSet dir="${custom-tasks.dir}/forbiddenApis">
+ <include name="jdk.txt" />
+ <include name="jdk-deprecated.txt" />
+ <include name="commons-io.txt" />
+ </apiFileSet>
+ <fileset dir="${basedir}/build">
+ <include name="**/*.class" />
+ <!-- exclude DIH for now as it is broken with Locales and Encodings: SOLR-1916 -->
+ <exclude name="contrib/solr-dataimporthandler*/**" />
+ </fileset>
+ </forbidden-apis>
+ </target>
+
+ <target name="-check-forbidden-test-apis">
+ <forbidden-apis apiFile="${custom-tasks.dir}/forbiddenApis/tests.txt">
+ <classpath refid="junit-path"/>
+ <fileset dir="${basedir}/build">
+ <include name="**/classes/test/**/*.class"/>
+ <include name="solr-test-framework/**/*.class"/>
+ <!-- not actually a test -->
+ <exclude name="solr-core/classes/test/org/apache/solr/search/DocSetPerf.class"/>
+ <!-- imported code -->
+ <exclude name="solr-core/classes/test/org/apache/solr/internal/**/*.class"/>
+ </fileset>
+ </forbidden-apis>
+ </target>
<!-- rat sources -->
<target name="rat-sources">
@@ -223,64 +258,26 @@
</delete>
</target>
- <!-- Clover targets -->
- <target name="clover" depends="clover.setup, clover.info"
- description="Instrument the Unit tests using Clover. Requires a Clover license and clover.jar in the ANT classpath. To use, specify -Drun.clover=true on the command line."/>
-
- <target name="clover.setup" if="clover.enabled">
- <taskdef resource="cloverlib.xml"/>
- <mkdir dir="${clover.db.dir}"/>
- <clover-setup initString="${clover.db.dir}/lucene_coverage.db">
- <fileset dir="core/src/java"/>
- <fileset dir="solrj/src/java"/>
- <fileset dir="contrib/analysis-extras/src/java"/>
- <fileset dir="contrib/clustering/src/java"/>
- <fileset dir="contrib/dataimporthandler/src/java"/>
- <fileset dir="contrib/dataimporthandler-extras/src/java"/>
- <fileset dir="contrib/extraction/src/java"/>
- <fileset dir="contrib/uima/src/java"/>
- <testsources dir="test-framework/src/java"/>
- <testsources dir="core/src/test"/>
- <testsources dir="solrj/src/test"/>
- <testsources dir="contrib/analysis-extras/src/test" />
- <testsources dir="contrib/clustering/src/test" />
- <testsources dir="contrib/dataimporthandler/src/test" />
- <testsources dir="contrib/dataimporthandler-extras/src/test" />
- <testsources dir="contrib/extraction/src/test" />
- <testsources dir="contrib/uima/src/test" />
- </clover-setup>
- </target>
-
- <target name="clover.info" unless="clover.present">
- <echo>
- Clover not found. Code coverage reports disabled.
- </echo>
- </target>
-
- <target name="clover.check">
- <fail unless="clover.present">
- ##################################################################
- Clover not found.
- Please make sure clover.jar is in ANT_HOME/lib, or made available
- to Ant using other mechanisms like -lib or CLASSPATH.
- ##################################################################
- </fail>
- </target>
-
- <!-- Run after Junit tests. -->
- <target name="generate-clover-reports" depends="clover.check, clover">
+ <!--
+ Run after Junit tests.
+ -->
+ <target name="generate-clover-reports" depends="clover">
+ <fail unless="run.clover">Clover not enabled!</fail>
<mkdir dir="${clover.report.dir}"/>
+ <fileset dir="build" id="clover.test.result.files">
+ <include name="**/test/TEST-*.xml"/>
+ </fileset>
<clover-report>
- <current outfile="${clover.report.dir}/clover.xml"
- title="${fullnamever}">
- <format type="xml"/>
- </current>
- <current outfile="${clover.report.dir}" title="${fullnamever}">
- <format type="html"/>
- </current>
+ <current outfile="${clover.report.dir}" title="${final.name}" numThreads="0">
+ <format type="html" filter="assert"/>
+ <testresults refid="clover.test.result.files"/>
+ </current>
+ <current outfile="${clover.report.dir}/clover.xml" title="${final.name}">
+ <format type="xml" filter="assert"/>
+ <testresults refid="clover.test.result.files"/>
+ </current>
</clover-report>
</target>
-
<!-- ========================================================================= -->
<!-- ===================== DISTRIBUTION-RELATED TASKS ======================== -->
@@ -340,15 +337,17 @@
<!-- Exclude javadoc package-list files under licenses incompatible with the ASL -->
<delete dir="${svn.export.dir}/lucene/tools/javadoc/java6"/>
+ <!-- Exclude clover license files incompatible with the ASL -->
+ <delete dir="${svn.export.dir}/lucene/tools/clover"/>
<tar destfile="${source.package.file}" compression="gzip" longfile="gnu">
<tarfileset dir="${svn.export.dir}"
prefix="${fullnamever}"
- excludes="example/**/*.sh example/**/bin/"/>
+ excludes="solr/example/**/*.sh solr/example/**/bin/ solr/scripts/**"/>
<tarfileset dir="${svn.export.dir}"
prefix="${fullnamever}"
filemode="755"
- includes="example/**/*.sh example/**/bin/"/>
+ includes="solr/example/**/*.sh solr/example/**/bin/ solr/scripts/**"/>
<tarfileset dir="${svn.export.dir}/solr" prefix="${fullnamever}"
includes="NOTICE.txt,LICENSE.txt"/>
</tar>
@@ -387,7 +386,7 @@
depends="init-dist, dist, example, javadocs">
<mkdir dir="${dest}/${fullnamever}"/>
<delete includeemptydirs="true">
- <fileset dir="${example}/work" includes="**/*"/>
+ <fileset dir="${example}/solr-webapp" includes="**/*"/>
<fileset dir="${dest}/${fullnamever}" includes="**/*"/>
</delete>
Modified: lucene/dev/branches/pforcodec_3892/solr/cloud-dev/example1.sh
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/cloud-dev/example1.sh?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/cloud-dev/example1.sh (original)
+++ lucene/dev/branches/pforcodec_3892/solr/cloud-dev/example1.sh Thu Jul 19 15:58:54 2012
@@ -15,7 +15,7 @@ cp -r -f example example2
cd example
-java -DzkRun -DnumShards=2 -DSTOP.PORT=7983 -DSTOP.KEY=key -Dbootstrap_confdir=solr/conf -jar start.jar 1>example.log 2>&1 &
+java -DzkRun -DnumShards=2 -DSTOP.PORT=7983 -DSTOP.KEY=key -Dbootstrap_conf=true -jar start.jar 1>example.log 2>&1 &
sleep 10
Modified: lucene/dev/branches/pforcodec_3892/solr/cloud-dev/example2.sh
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/cloud-dev/example2.sh?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/cloud-dev/example2.sh (original)
+++ lucene/dev/branches/pforcodec_3892/solr/cloud-dev/example2.sh Thu Jul 19 15:58:54 2012
@@ -19,8 +19,9 @@ cp -r -f example example4
cd example
-java -DzkRun -DnumShards=2 -DSTOP.PORT=7983 -DSTOP.KEY=key -Dbootstrap_confdir=solr/conf -jar start.jar 1>example.log 2>&1 &
+java -DzkRun -DnumShards=2 -DSTOP.PORT=7983 -DSTOP.KEY=key -Dbootstrap_conf=true -jar start.jar 1>example.log 2>&1 &
+# wait for config to go up
sleep 10
cd ../example2
Modified: lucene/dev/branches/pforcodec_3892/solr/cloud-dev/example3.sh
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/cloud-dev/example3.sh?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/cloud-dev/example3.sh (original)
+++ lucene/dev/branches/pforcodec_3892/solr/cloud-dev/example3.sh Thu Jul 19 15:58:54 2012
@@ -19,10 +19,10 @@ cp -r -f example example4
cd example
-java -DzkRun -DnumShards=2 -DSTOP.PORT=7983 -DSTOP.KEY=key -Dbootstrap_confdir=solr/conf -DzkHost=localhost:9983,localhost:14574,localhost:14585 -jar start.jar 1>example.log 2>&1 &
+java -DzkRun -DnumShards=2 -DSTOP.PORT=7983 -DSTOP.KEY=key -Dbootstrap_conf=true -DzkHost=localhost:9983,localhost:14574,localhost:14585 -jar start.jar 1>example.log 2>&1 &
cd ../example2
-java -Djetty.port=13574 -DzkRun -DzkHost=localhost:9983,localhost:14574,localhost:14575 -DSTOP.PORT=6574 -DSTOP.KEY=key -jar start.jar 1>example2.log 2>&1 &
+java -Djetty.port=13574 -DzkRun -DzkHost=localhost:9983,localhost:14574,localhost:14585 -DSTOP.PORT=6574 -DSTOP.KEY=key -jar start.jar 1>example2.log 2>&1 &
cd ../example3
java -Djetty.port=13585 -DzkRun -DzkHost=localhost:9983,localhost:14574,localhost:14585 -DSTOP.PORT=6575 -DSTOP.KEY=key -jar start.jar 1>example3.log 2>&1 &
Modified: lucene/dev/branches/pforcodec_3892/solr/cloud-dev/solrcloud-multi-start.sh
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/cloud-dev/solrcloud-multi-start.sh?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/cloud-dev/solrcloud-multi-start.sh (original)
+++ lucene/dev/branches/pforcodec_3892/solr/cloud-dev/solrcloud-multi-start.sh Thu Jul 19 15:58:54 2012
@@ -24,7 +24,7 @@ cp -r -f example example4
cp -r -f example example5
cp -r -f example example6
-java -classpath lib/*:dist/*:build/lucene-libs/* org.apache.solr.cloud.ZkController 127.0.0.1:9983 example/multicore 8983
+java -classpath lib/*:dist/*:build/lucene-libs/* org.apache.solr.cloud.ZkCLI -cmd upconf -zkhost 127.0.0.1:9983 -solrhome example/multicore -runzk 8983
cd example
java -DzkRun -DnumShards=2 -DSTOP.PORT=7983 -DSTOP.KEY=key -Dsolr.solr.home=multicore -jar start.jar 1>example.log 2>&1 &
Modified: lucene/dev/branches/pforcodec_3892/solr/cloud-dev/solrcloud-start.sh
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/cloud-dev/solrcloud-start.sh?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/cloud-dev/solrcloud-start.sh (original)
+++ lucene/dev/branches/pforcodec_3892/solr/cloud-dev/solrcloud-start.sh Thu Jul 19 15:58:54 2012
@@ -22,7 +22,7 @@ cp -r -f example example4
cp -r -f example example5
cp -r -f example example6
-java -classpath lib/*:dist/*:build/lucene-libs/* org.apache.solr.cloud.ZkController 127.0.0.1:9983 example/solr 8983
+java -classpath lib/*:dist/*:build/lucene-libs/* org.apache.solr.cloud.ZkCLI -cmd bootstrap -zkhost 127.0.0.1:9983 -solrhome example/solr -runzk 8983
cd example
java -DzkRun -DnumShards=2 -DSTOP.PORT=7983 -DSTOP.KEY=key -jar start.jar 1>example.log 2>&1 &
Modified: lucene/dev/branches/pforcodec_3892/solr/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/common-build.xml?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/common-build.xml (original)
+++ lucene/dev/branches/pforcodec_3892/solr/common-build.xml Thu Jul 19 15:58:54 2012
@@ -49,15 +49,7 @@
<property name="tests.loggingfile" value="${common-solr.dir}/testlogging.properties"/>
<property name="tests.cleanthreads.sysprop" value="perClass"/>
- <property name="clover.db.dir" location="${dest}/test/clover/db"/>
- <property name="clover.report.dir" location="${dest}/test/clover/reports"/>
- <available property="clover.present" classname="com.cenqua.clover.tasks.CloverReportTask"/>
- <condition property="clover.enabled">
- <and>
- <isset property="run.clover"/>
- <isset property="clover.present"/>
- </and>
- </condition>
+ <property name="clover.report.dir" location="${dest}/clover/reports"/>
<import file="${common-solr.dir}/../lucene/module-build.xml"/>
Modified: lucene/dev/branches/pforcodec_3892/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/MorfologikFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/MorfologikFilterFactory.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/MorfologikFilterFactory.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/MorfologikFilterFactory.java Thu Jul 19 15:58:54 2012
@@ -69,7 +69,7 @@ public class MorfologikFilterFactory ext
String dictionaryName = args.get(DICTIONARY_SCHEMA_ATTRIBUTE);
if (dictionaryName != null && !dictionaryName.isEmpty()) {
try {
- DICTIONARY dictionary = DICTIONARY.valueOf(dictionaryName.toUpperCase(Locale.ENGLISH));
+ DICTIONARY dictionary = DICTIONARY.valueOf(dictionaryName.toUpperCase(Locale.ROOT));
assert dictionary != null;
this.dictionary = dictionary;
} catch (IllegalArgumentException e) {
Modified: lucene/dev/branches/pforcodec_3892/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestMorfologikFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestMorfologikFilterFactory.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestMorfologikFilterFactory.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestMorfologikFilterFactory.java Thu Jul 19 15:58:54 2012
@@ -7,7 +7,6 @@ import java.util.Map;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
-import org.apache.solr.schema.IndexSchema;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
Modified: lucene/dev/branches/pforcodec_3892/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java Thu Jul 19 15:58:54 2012
@@ -67,13 +67,13 @@ public class TestICUCollationField exten
tmpFile.mkdir();
// make data and conf dirs
- new File(tmpFile, "data").mkdir();
- File confDir = new File(tmpFile, "conf");
- confDir.mkdir();
+ new File(tmpFile + "/collection1", "data").mkdirs();
+ File confDir = new File(tmpFile + "/collection1", "conf");
+ confDir.mkdirs();
// copy over configuration files
- FileUtils.copyFile(getFile("analysis-extras/solr/conf/solrconfig-icucollate.xml"), new File(confDir, "solrconfig.xml"));
- FileUtils.copyFile(getFile("analysis-extras/solr/conf/schema-icucollate.xml"), new File(confDir, "schema.xml"));
+ FileUtils.copyFile(getFile("analysis-extras/solr/collection1/conf/solrconfig-icucollate.xml"), new File(confDir, "solrconfig.xml"));
+ FileUtils.copyFile(getFile("analysis-extras/solr/collection1/conf/schema-icucollate.xml"), new File(confDir, "schema.xml"));
// generate custom collation rules (DIN 5007-2), saving to customrules.dat
RuleBasedCollator baseCollator = (RuleBasedCollator) Collator.getInstance(new ULocale("de", "DE"));
Modified: lucene/dev/branches/pforcodec_3892/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java Thu Jul 19 15:58:54 2012
@@ -147,7 +147,7 @@ public class CarrotClusteringEngine exte
final IResource foundResource = new IResource() {
@Override
- public InputStream open() throws IOException {
+ public InputStream open() {
return new ByteArrayInputStream(asBytes);
}
Modified: lucene/dev/branches/pforcodec_3892/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2StemmerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2StemmerFactory.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2StemmerFactory.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2StemmerFactory.java Thu Jul 19 15:58:54 2012
@@ -180,7 +180,7 @@ public class LuceneCarrot2StemmerFactory
private char[] buffer = new char[0];
- private LuceneStemmerAdapter() throws Exception {
+ private LuceneStemmerAdapter() {
delegate = new org.apache.lucene.analysis.ar.ArabicStemmer();
normalizer = new org.apache.lucene.analysis.ar.ArabicNormalizer();
}
Modified: lucene/dev/branches/pforcodec_3892/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2TokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2TokenizerFactory.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2TokenizerFactory.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2TokenizerFactory.java Thu Jul 19 15:58:54 2012
@@ -141,9 +141,9 @@ public class LuceneCarrot2TokenizerFacto
array.reset(term.buffer(), 0, term.length());
}
- public void reset(Reader input) throws IOException {
+ public void reset(Reader input) {
try {
- sentenceTokenizer.reset(input);
+ sentenceTokenizer.setReader(input);
wordTokenFilter = (TokenStream) tokenFilterClass.getConstructor(
TokenStream.class).newInstance(sentenceTokenizer);
term = wordTokenFilter.addAttribute(CharTermAttribute.class);
Modified: lucene/dev/branches/pforcodec_3892/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/DistributedClusteringComponentTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/DistributedClusteringComponentTest.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/DistributedClusteringComponentTest.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/DistributedClusteringComponentTest.java Thu Jul 19 15:58:54 2012
@@ -25,7 +25,7 @@ public class DistributedClusteringCompon
@Override
public String getSolrHome() {
- return "clustering/solr";
+ return getFile("clustering/solr/collection1").getParent();
}
@Override
Modified: lucene/dev/branches/pforcodec_3892/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/DuplicatingTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/DuplicatingTokenizerFactory.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/DuplicatingTokenizerFactory.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/DuplicatingTokenizerFactory.java Thu Jul 19 15:58:54 2012
@@ -39,7 +39,7 @@ public class DuplicatingTokenizerFactory
}
@Override
- public void reset(Reader input) throws IOException {
+ public void reset(Reader input) {
delegate.reset(input);
}
Modified: lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/MailEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/MailEntityProcessor.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/MailEntityProcessor.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/MailEntityProcessor.java Thu Jul 19 15:58:54 2012
@@ -171,7 +171,7 @@ public class MailEntityProcessor extends
InputStream is = part.getInputStream();
String fileName = part.getFileName();
Metadata md = new Metadata();
- md.set(HttpHeaders.CONTENT_TYPE, ctype.getBaseType().toLowerCase(Locale.ENGLISH));
+ md.set(HttpHeaders.CONTENT_TYPE, ctype.getBaseType().toLowerCase(Locale.ROOT));
md.set(TikaMetadataKeys.RESOURCE_NAME_KEY, fileName);
String content = tika.parseToString(is, md);
if (disp != null && disp.equalsIgnoreCase(Part.ATTACHMENT)) {
Modified: lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java Thu Jul 19 15:58:54 2012
@@ -128,7 +128,7 @@ public class DataImporter {
private void loadSchemaFieldMap() {
Map<String, SchemaField> modLnvsf = new HashMap<String, SchemaField>();
for (Map.Entry<String, SchemaField> entry : schema.getFields().entrySet()) {
- modLnvsf.put(entry.getKey().toLowerCase(Locale.ENGLISH), entry.getValue());
+ modLnvsf.put(entry.getKey().toLowerCase(Locale.ROOT), entry.getValue());
}
lowerNameVsSchemaField = Collections.unmodifiableMap(modLnvsf);
}
@@ -139,7 +139,7 @@ public class DataImporter {
schemaField = schema.getFieldOrNull(caseInsensitiveName);
}
if (schemaField == null) {
- schemaField = lowerNameVsSchemaField.get(caseInsensitiveName.toLowerCase(Locale.ENGLISH));
+ schemaField = lowerNameVsSchemaField.get(caseInsensitiveName.toLowerCase(Locale.ROOT));
}
return schemaField;
}
Modified: lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/HTMLStripTransformer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/HTMLStripTransformer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/HTMLStripTransformer.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/HTMLStripTransformer.java Thu Jul 19 15:58:54 2012
@@ -17,7 +17,6 @@
package org.apache.solr.handler.dataimport;
import org.apache.lucene.analysis.charfilter.HTMLStripCharFilter;
-import org.apache.lucene.analysis.CharReader;
import java.io.IOException;
import java.io.StringReader;
@@ -73,7 +72,7 @@ public class HTMLStripTransformer extend
StringBuilder out = new StringBuilder();
StringReader strReader = new StringReader(value);
try {
- HTMLStripCharFilter html = new HTMLStripCharFilter(CharReader.get(strReader.markSupported() ? strReader : new BufferedReader(strReader)));
+ HTMLStripCharFilter html = new HTMLStripCharFilter(strReader.markSupported() ? strReader : new BufferedReader(strReader));
char[] cbuf = new char[1024 * 10];
while (true) {
int count = html.read(cbuf);
Modified: lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java Thu Jul 19 15:58:54 2012
@@ -72,7 +72,7 @@ public class NumberFormatTransformer ext
}
Object val = row.get(srcCol);
- String styleSmall = style.toLowerCase(Locale.ENGLISH);
+ String styleSmall = style.toLowerCase(Locale.ROOT);
if (val instanceof List) {
List<String> inputs = (List) val;
Modified: lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java Thu Jul 19 15:58:54 2012
@@ -68,7 +68,7 @@ public abstract class AbstractDataImport
@After
public void tearDown() throws Exception {
// remove dataimport.properties
- File f = new File("solr/conf/dataimport.properties");
+ File f = new File("solr/collection1/conf/dataimport.properties");
log.info("Looking for dataimport.properties at: " + f.getAbsolutePath());
if (f.exists()) {
log.info("Deleting dataimport.properties");
Modified: lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestCachedSqlEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestCachedSqlEntityProcessor.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestCachedSqlEntityProcessor.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestCachedSqlEntityProcessor.java Thu Jul 19 15:58:54 2012
@@ -108,7 +108,7 @@ public class TestCachedSqlEntityProcesso
if (r == null)
break;
rows.add(r);
- assertEquals(r.get("desc").toString().toUpperCase(Locale.ENGLISH), r.get("desc"));
+ assertEquals(r.get("desc").toString().toUpperCase(Locale.ROOT), r.get("desc"));
}
assertEquals(2, rows.size());
assertEquals(2, rows.get(0).size());
@@ -176,7 +176,7 @@ public class TestCachedSqlEntityProcesso
Object val = entry.getValue();
if (val instanceof String) {
String s = (String) val;
- entry.setValue(s.toUpperCase(Locale.ENGLISH));
+ entry.setValue(s.toUpperCase(Locale.ROOT));
}
}
return row;
Modified: lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestContentStreamDataSource.java Thu Jul 19 15:58:54 2012
@@ -40,7 +40,7 @@ import java.util.List;
* @since solr 1.4
*/
public class TestContentStreamDataSource extends AbstractDataImportHandlerTestCase {
- private static final String CONF_DIR = "dih/solr/conf/";
+ private static final String CONF_DIR = "dih/solr/collection1/conf/";
SolrInstance instance = null;
JettySolrRunner jetty;
@@ -151,8 +151,8 @@ public class TestContentStreamDataSource
homeDir = new File(home, "inst");
- dataDir = new File(homeDir, "data");
- confDir = new File(homeDir, "conf");
+ dataDir = new File(homeDir + "/collection1", "data");
+ confDir = new File(homeDir + "/collection1", "conf");
homeDir.mkdirs();
dataDir.mkdirs();
Modified: lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestErrorHandling.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestErrorHandling.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestErrorHandling.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestErrorHandling.java Thu Jul 19 15:58:54 2012
@@ -46,7 +46,7 @@ public class TestErrorHandling extends A
}
@AfterClass
- public static void afterClass() throws Exception {
+ public static void afterClass() {
if (savedFactory == null) {
System.clearProperty("solr.directoryFactory");
} else {
Modified: lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java Thu Jul 19 15:58:54 2012
@@ -51,7 +51,7 @@ public class TestSolrEntityProcessorEndT
private static final String SOLR_CONFIG = "dataimport-solrconfig.xml";
private static final String SOLR_SCHEMA = "dataimport-schema.xml";
private static final String SOLR_HOME = "dih/solr";
- private static final String CONF_DIR = "dih" + File.separator + "solr" + File.separator + "conf" + File.separator;
+ private static final String CONF_DIR = "dih" + File.separator + "solr" + File.separator + "collection1" + File.separator + "conf" + File.separator;
private static final List<Map<String,Object>> DB_DOCS = new ArrayList<Map<String,Object>>();
private static final List<Map<String,Object>> SOLR_DOCS = new ArrayList<Map<String,Object>>();
@@ -101,13 +101,13 @@ public class TestSolrEntityProcessorEndT
//TODO: fix this test to close its directories
static String savedFactory;
@BeforeClass
- public static void beforeClass() throws Exception {
+ public static void beforeClass() {
savedFactory = System.getProperty("solr.DirectoryFactory");
System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory");
}
@AfterClass
- public static void afterClass() throws Exception {
+ public static void afterClass() {
if (savedFactory == null) {
System.clearProperty("solr.directoryFactory");
} else {
@@ -308,8 +308,8 @@ public class TestSolrEntityProcessorEndT
+ System.currentTimeMillis());
homeDir = new File(home + "inst");
- dataDir = new File(homeDir, "data");
- confDir = new File(homeDir, "conf");
+ dataDir = new File(homeDir + "/collection1", "data");
+ confDir = new File(homeDir + "/collection1", "conf");
homeDir.mkdirs();
dataDir.mkdirs();
Modified: lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java Thu Jul 19 15:58:54 2012
@@ -22,7 +22,6 @@ import org.junit.BeforeClass;
import org.junit.Test;
import java.io.File;
-import java.io.FileOutputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -70,7 +69,7 @@ public class TestSqlEntityProcessorDelta
}
@AfterClass
- public static void afterClass() throws Exception {
+ public static void afterClass() {
if (savedFactory == null) {
System.clearProperty("solr.directoryFactory");
} else {
Modified: lucene/dev/branches/pforcodec_3892/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java Thu Jul 19 15:58:54 2012
@@ -44,6 +44,7 @@ import org.apache.tika.parser.AutoDetect
import org.apache.tika.parser.DefaultParser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
+import org.apache.tika.parser.PasswordProvider;
import org.apache.tika.sax.XHTMLContentHandler;
import org.apache.tika.sax.xpath.Matcher;
import org.apache.tika.sax.xpath.MatchingContentHandler;
@@ -90,7 +91,6 @@ public class ExtractingDocumentLoader ex
protected TikaConfig config;
protected SolrContentHandlerFactory factory;
- //protected Collection<String> dateFormats = DateUtil.DEFAULT_DATE_FORMATS;
public ExtractingDocumentLoader(SolrQueryRequest req, UpdateRequestProcessor processor,
TikaConfig config, SolrContentHandlerFactory factory) {
@@ -114,8 +114,6 @@ public class ExtractingDocumentLoader ex
/**
* this must be MT safe... may be called concurrently from multiple threads.
*
- * @param
- * @param
*/
void doAdd(SolrContentHandler handler, AddUpdateCommand template)
throws IOException {
@@ -140,7 +138,7 @@ public class ExtractingDocumentLoader ex
String streamType = req.getParams().get(ExtractingParams.STREAM_TYPE, null);
if (streamType != null) {
//Cache? Parsers are lightweight to construct and thread-safe, so I'm told
- MediaType mt = MediaType.parse(streamType.trim().toLowerCase(Locale.ENGLISH));
+ MediaType mt = MediaType.parse(streamType.trim().toLowerCase(Locale.ROOT));
parser = new DefaultParser(config.getMediaTypeRegistry()).getParsers().get(mt);
} else {
parser = autoDetectParser;
@@ -206,6 +204,23 @@ public class ExtractingDocumentLoader ex
try{
//potentially use a wrapper handler for parsing, but we still need the SolrContentHandler for getting the document.
ParseContext context = new ParseContext();//TODO: should we design a way to pass in parse context?
+
+ // Password handling
+ RegexRulesPasswordProvider epp = new RegexRulesPasswordProvider();
+ String pwMapFile = params.get(ExtractingParams.PASSWORD_MAP_FILE);
+ if(pwMapFile != null && pwMapFile.length() > 0) {
+ InputStream is = req.getCore().getResourceLoader().openResource(pwMapFile);
+ if(is != null) {
+ log.debug("Password file supplied: "+pwMapFile);
+ epp.parse(is);
+ }
+ }
+ context.set(PasswordProvider.class, epp);
+ String resourcePassword = params.get(ExtractingParams.RESOURCE_PASSWORD);
+ if(resourcePassword != null) {
+ epp.setExplicitPassword(resourcePassword);
+ log.debug("Literal password supplied for file "+resourceName);
+ }
parser.parse(inputStream, parsingHandler, metadata, context);
} catch (TikaException e) {
if(ignoreTikaException)
Modified: lucene/dev/branches/pforcodec_3892/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java Thu Jul 19 15:58:54 2012
@@ -96,6 +96,10 @@ public interface ExtractingParams {
*/
public static final String CAPTURE_ATTRIBUTES = "captureAttr";
+ /**
+ * Literal field values will by default override other values such as metadata and content. Set this to false to revert to pre-4.0 behaviour
+ */
+ public static final String LITERALS_OVERRIDE = "literalsOverride";
/**
* Capture the specified fields (and everything included below it that isn't capture by some other capture field) separately from the default. This is different
@@ -132,6 +136,10 @@ public interface ExtractingParams {
*/
public static final String RESOURCE_NAME = "resource.name";
+ /**
+ * Optional. The password for this resource. Will be used instead of the rule based password lookup mechanisms
+ */
+ public static final String RESOURCE_PASSWORD = "resource.password";
/**
* Optional. If specified, the prefix will be prepended to all Metadata, such that it would be possible
@@ -144,4 +152,14 @@ public interface ExtractingParams {
* will be used instead.
*/
public static final String DEFAULT_FIELD = "defaultField";
+
+ /**
+ * Optional. If specified, loads the file as a source for password lookups for Tika encrypted documents.
+ * <p>
+ * File format is Java properties format with one key=value per line.
+ * The key is evaluated as a regex against the file name, and the value is the password
+ * The rules are evaluated top-bottom, i.e. the first match will be used
+ * If you want a fallback password to be always used, supply a .*=<defaultmypassword> at the end
+ */
+ public static final String PASSWORD_MAP_FILE = "passwordsFile";
}
Modified: lucene/dev/branches/pforcodec_3892/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java Thu Jul 19 15:58:54 2012
@@ -66,6 +66,9 @@ public class SolrContentHandler extends
protected String unknownFieldPrefix = "";
protected String defaultField = "";
+ private boolean literalsOverride;
+ private Set<String> literalFieldNames;
+
public SolrContentHandler(Metadata metadata, SolrParams params, IndexSchema schema) {
this(metadata, params, schema, DateUtil.DEFAULT_DATE_FORMATS);
}
@@ -81,6 +84,7 @@ public class SolrContentHandler extends
this.lowerNames = params.getBool(LOWERNAMES, false);
this.captureAttribs = params.getBool(CAPTURE_ATTRIBUTES, false);
+ this.literalsOverride = params.getBool(LITERALS_OVERRIDE, true);
this.unknownFieldPrefix = params.get(UNKNOWN_FIELD_PREFIX, "");
this.defaultField = params.get(DEFAULT_FIELD, "");
String[] captureFields = params.getParams(CAPTURE_ELEMENTS);
@@ -107,13 +111,11 @@ public class SolrContentHandler extends
* @see #addLiterals()
*/
public SolrInputDocument newDocument() {
- float boost = 1.0f;
- //handle the metadata extracted from the document
- addMetadata();
-
- //handle the literals from the params
+ //handle the literals from the params. NOTE: This MUST be called before the others in order for literals to override other values
addLiterals();
+ //handle the metadata extracted from the document
+ addMetadata();
//add in the content
addContent();
@@ -134,8 +136,10 @@ public class SolrContentHandler extends
protected void addCapturedContent() {
for (Map.Entry<String, StringBuilder> entry : fieldBuilders.entrySet()) {
if (entry.getValue().length() > 0) {
- addField(entry.getKey(), entry.getValue().toString(), null);
- }
+ String fieldName = entry.getKey();
+ if (literalsOverride && literalFieldNames.contains(fieldName))
+ continue;
+ addField(fieldName, entry.getValue().toString(), null); }
}
}
@@ -144,6 +148,8 @@ public class SolrContentHandler extends
* and the {@link #catchAllBuilder}
*/
protected void addContent() {
+ if (literalsOverride && literalFieldNames.contains(contentFieldName))
+ return;
addField(contentFieldName, catchAllBuilder.toString(), null);
}
@@ -152,12 +158,14 @@ public class SolrContentHandler extends
*/
protected void addLiterals() {
Iterator<String> paramNames = params.getParameterNamesIterator();
+ literalFieldNames = new HashSet<String>();
while (paramNames.hasNext()) {
String pname = paramNames.next();
if (!pname.startsWith(LITERALS_PREFIX)) continue;
String name = pname.substring(LITERALS_PREFIX.length());
addField(name, null, params.getParams(pname));
+ literalFieldNames.add(name);
}
}
@@ -166,6 +174,8 @@ public class SolrContentHandler extends
*/
protected void addMetadata() {
for (String name : metadata.names()) {
+ if (literalsOverride && literalFieldNames.contains(name))
+ continue;
String[] vals = metadata.getValues(name);
addField(name, null, vals);
}
Modified: lucene/dev/branches/pforcodec_3892/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java Thu Jul 19 15:58:54 2012
@@ -394,7 +394,7 @@ public class ExtractingRequestHandlerTes
try{
loadLocal("extraction/password-is-solrcell.docx",
"literal.id", "one");
- fail("TikaException is expected because of trying to extract text from password protected word file.");
+ fail("TikaException is expected because of trying to extract text from password protected word file without supplying a password.");
}
catch(Exception expected){}
assertU(commit());
@@ -444,6 +444,138 @@ public class ExtractingRequestHandlerTes
}
catch(Exception expected){}
}
+
+ public void testLiteralsOverride() throws Exception {
+ ExtractingRequestHandler handler = (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");
+ assertTrue("handler is null and it shouldn't be", handler != null);
+
+ assertQ(req("*:*"), "//*[@numFound='0']");
+
+ // Here Tika should parse out a title for this document:
+ loadLocal("extraction/solr-word.pdf",
+ "fmap.created", "extractedDate",
+ "fmap.producer", "extractedProducer",
+ "fmap.creator", "extractedCreator",
+ "fmap.Keywords", "extractedKeywords",
+ "fmap.Author", "extractedAuthor",
+ "literal.id", "three",
+ "fmap.content", "extractedContent",
+ "fmap.language", "extractedLanguage",
+ "fmap.Creation-Date", "extractedDate",
+ "fmap.AAPL:Keywords", "ignored_a",
+ "fmap.xmpTPg:NPages", "ignored_a",
+ "fmap.Last-Modified", "extractedDate");
+
+ // Here the literal value should override the Tika-parsed title:
+ loadLocal("extraction/solr-word.pdf",
+ "literal.title", "wolf-man",
+ "fmap.created", "extractedDate",
+ "fmap.producer", "extractedProducer",
+ "fmap.creator", "extractedCreator",
+ "fmap.Keywords", "extractedKeywords",
+ "fmap.Author", "extractedAuthor",
+ "literal.id", "four",
+ "fmap.content", "extractedContent",
+ "fmap.language", "extractedLanguage",
+ "fmap.Creation-Date", "extractedDate",
+ "fmap.AAPL:Keywords", "ignored_a",
+ "fmap.xmpTPg:NPages", "ignored_a",
+ "fmap.Last-Modified", "extractedDate");
+
+ // Here we mimic the old behaviour where literals are added, not overridden
+ loadLocal("extraction/solr-word.pdf",
+ "literalsOverride", "false",
+ // Trick - we first map the metadata-title to an ignored field before we replace with literal title
+ "fmap.title", "ignored_a",
+ "literal.title", "old-behaviour",
+ "literal.extractedKeywords", "literalkeyword",
+ "fmap.created", "extractedDate",
+ "fmap.producer", "extractedProducer",
+ "fmap.creator", "extractedCreator",
+ "fmap.Keywords", "extractedKeywords",
+ "fmap.Author", "extractedAuthor",
+ "literal.id", "five",
+ "fmap.content", "extractedContent",
+ "fmap.language", "extractedLanguage",
+ "fmap.Creation-Date", "extractedDate",
+ "fmap.AAPL:Keywords", "ignored_a",
+ "fmap.xmpTPg:NPages", "ignored_a",
+ "fmap.Last-Modified", "extractedDate");
+
+ assertU(commit());
+
+ assertQ(req("title:solr-word"), "//*[@numFound='1']");
+ assertQ(req("title:wolf-man"), "//*[@numFound='1']");
+ assertQ(req("extractedKeywords:(solr AND word AND pdf AND literalkeyword)"), "//*[@numFound='1']");
+ }
+
+ @Test
+ public void testPasswordProtected() throws Exception {
+ // PDF, Passwords from resource.password
+ loadLocal("extraction/enctypted-password-is-solrRules.pdf",
+ "fmap.created", "extractedDate",
+ "fmap.producer", "extractedProducer",
+ "fmap.creator", "extractedCreator",
+ "fmap.Keywords", "extractedKeywords",
+ "fmap.Creation-Date", "extractedDate",
+ "uprefix", "ignored_",
+ "fmap.Author", "extractedAuthor",
+ "fmap.content", "wdf_nocase",
+ "literal.id", "pdfpwliteral",
+ "resource.name", "enctypted-password-is-solrRules.pdf",
+ "resource.password", "solrRules",
+ "fmap.Last-Modified", "extractedDate");
+
+ // PDF, Passwords from passwords property file
+ loadLocal("extraction/enctypted-password-is-solrRules.pdf",
+ "fmap.created", "extractedDate",
+ "fmap.producer", "extractedProducer",
+ "fmap.creator", "extractedCreator",
+ "fmap.Keywords", "extractedKeywords",
+ "fmap.Creation-Date", "extractedDate",
+ "uprefix", "ignored_",
+ "fmap.Author", "extractedAuthor",
+ "fmap.content", "wdf_nocase",
+ "literal.id", "pdfpwfile",
+ "resource.name", "enctypted-password-is-solrRules.pdf",
+ "passwordsFile", "passwordRegex.properties", // Passwords-file
+ "fmap.Last-Modified", "extractedDate");
+
+ // DOCX, Explicit password
+ loadLocal("extraction/password-is-Word2010.docx",
+ "fmap.created", "extractedDate",
+ "fmap.producer", "extractedProducer",
+ "fmap.creator", "extractedCreator",
+ "fmap.Keywords", "extractedKeywords",
+ "fmap.Creation-Date", "extractedDate",
+ "fmap.Author", "extractedAuthor",
+ "fmap.content", "wdf_nocase",
+ "uprefix", "ignored_",
+ "literal.id", "docxpwliteral",
+ "resource.name", "password-is-Word2010.docx",
+ "resource.password", "Word2010", // Explicit password
+ "fmap.Last-Modified", "extractedDate");
+
+ // DOCX, Passwords from file
+ loadLocal("extraction/password-is-Word2010.docx",
+ "fmap.created", "extractedDate",
+ "fmap.producer", "extractedProducer",
+ "fmap.creator", "extractedCreator",
+ "fmap.Keywords", "extractedKeywords",
+ "fmap.Creation-Date", "extractedDate",
+ "uprefix", "ignored_",
+ "fmap.Author", "extractedAuthor",
+ "fmap.content", "wdf_nocase",
+ "literal.id", "docxpwfile",
+ "resource.name", "password-is-Word2010.docx",
+ "passwordsFile", "passwordRegex.properties", // Passwords-file
+ "fmap.Last-Modified", "extractedDate");
+
+ assertU(commit());
+ Thread.sleep(100);
+ assertQ(req("wdf_nocase:\"This is a test of PDF\""), "//*[@numFound='2']");
+ assertQ(req("wdf_nocase:\"Test password protected word doc\""), "//*[@numFound='2']");
+ }
SolrQueryResponse loadLocal(String filename, String... args) throws Exception {
LocalSolrQueryRequest req = (LocalSolrQueryRequest) req(args);
Modified: lucene/dev/branches/pforcodec_3892/solr/contrib/uima/src/test-files/uima/uima-tokenizers-solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/contrib/uima/src/test-files/uima/uima-tokenizers-solrconfig.xml?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/contrib/uima/src/test-files/uima/uima-tokenizers-solrconfig.xml (original)
+++ lucene/dev/branches/pforcodec_3892/solr/contrib/uima/src/test-files/uima/uima-tokenizers-solrconfig.xml Thu Jul 19 15:58:54 2012
@@ -400,7 +400,7 @@
<!--
DisMaxRequestHandler allows easy searching across multiple fields
for simple user-entered phrases. It's implementation is now just the
- standard SearchHandler with a default query type of "dismax". see
+ standard SearchHandler with a default query parser of "dismax". see
http://wiki.apache.org/solr/DisMaxRequestHandler
-->
<requestHandler name="dismax" class="solr.SearchHandler">
Modified: lucene/dev/branches/pforcodec_3892/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/contrib/uima/src/test/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java Thu Jul 19 15:58:54 2012
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
+import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.MultiMapSolrParams;
@@ -46,6 +47,7 @@ import org.junit.Test;
*
*
*/
+@Slow
public class UIMAUpdateRequestProcessorTest extends SolrTestCaseJ4 {
@BeforeClass
Modified: lucene/dev/branches/pforcodec_3892/solr/core/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/ivy.xml?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/ivy.xml (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/ivy.xml Thu Jul 19 15:58:54 2012
@@ -22,6 +22,7 @@
<dependencies>
<dependency org="commons-codec" name="commons-codec" rev="1.6" transitive="false"/>
<dependency org="commons-fileupload" name="commons-fileupload" rev="1.2.1" transitive="false"/>
+ <dependency org="commons-cli" name="commons-cli" rev="1.2" transitive="false"/>
<dependency org="org.apache.httpcomponents" name="httpcore" rev="4.1.4" transitive="false"/>
<dependency org="org.apache.httpcomponents" name="httpclient" rev="4.1.3" transitive="false"/>
<dependency org="org.apache.httpcomponents" name="httpmime" rev="4.1.3" transitive="false"/>
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/doc-files/tutorial.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/doc-files/tutorial.html?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/doc-files/tutorial.html (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/doc-files/tutorial.html Thu Jul 19 15:58:54 2012
@@ -656,7 +656,7 @@ to the final terms produced by the Query
<li>Make a copy of the Solr <span class="codefrag">example</span> directory as a template for your project.</li>
-<li>Customize the schema and other config in <span class="codefrag">solr/conf/</span> to meet your needs.</li>
+<li>Customize the schema and other config in <span class="codefrag">solr/collection1/conf/</span> to meet your needs.</li>
</ul>
<p>
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/HTMLStripCharFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/HTMLStripCharFilterFactory.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/HTMLStripCharFilterFactory.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/HTMLStripCharFilterFactory.java Thu Jul 19 15:58:54 2012
@@ -18,10 +18,10 @@ package org.apache.solr.analysis;
* limitations under the License.
*/
-import org.apache.lucene.analysis.CharStream;
import org.apache.lucene.analysis.charfilter.HTMLStripCharFilter;
import org.apache.lucene.analysis.util.CharFilterFactory;
+import java.io.Reader;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
@@ -44,7 +44,7 @@ import java.util.regex.Pattern;
Set<String> escapedTags = null;
Pattern TAG_NAME_PATTERN = Pattern.compile("[^\\s,]+");
- public HTMLStripCharFilter create(CharStream input) {
+ public HTMLStripCharFilter create(Reader input) {
HTMLStripCharFilter charFilter;
if (null == escapedTags) {
charFilter = new HTMLStripCharFilter(input);
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/HunspellStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/HunspellStemFilterFactory.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/HunspellStemFilterFactory.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/HunspellStemFilterFactory.java Thu Jul 19 15:58:54 2012
@@ -28,6 +28,7 @@ import org.apache.lucene.analysis.util.I
import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
import org.apache.lucene.analysis.util.TokenFilterFactory;
+import org.apache.lucene.util.IOUtils;
/**
* TokenFilterFactory that creates instances of {@link org.apache.lucene.analysis.hunspell.HunspellStemFilter}.
@@ -76,7 +77,6 @@ public class HunspellStemFilterFactory e
else throw new InitializationException("Unknown value for " + PARAM_IGNORE_CASE + ": " + pic + ". Must be true or false");
}
-
String strictAffixParsingParam = args.get(PARAM_STRICT_AFFIX_PARSING);
boolean strictAffixParsing = true;
if(strictAffixParsingParam != null) {
@@ -85,14 +85,22 @@ public class HunspellStemFilterFactory e
else throw new InitializationException("Unknown value for " + PARAM_STRICT_AFFIX_PARSING + ": " + strictAffixParsingParam + ". Must be true or false");
}
+ InputStream affix = null;
+ List<InputStream> dictionaries = new ArrayList<InputStream>();
+
try {
- List<InputStream> dictionaries = new ArrayList<InputStream>();
+ dictionaries = new ArrayList<InputStream>();
for (String file : dictionaryFiles) {
dictionaries.add(loader.openResource(file));
}
- this.dictionary = new HunspellDictionary(loader.openResource(affixFile), dictionaries, luceneMatchVersion, ignoreCase, strictAffixParsing);
+ affix = loader.openResource(affixFile);
+
+ this.dictionary = new HunspellDictionary(affix, dictionaries, luceneMatchVersion, ignoreCase, strictAffixParsing);
} catch (Exception e) {
throw new InitializationException("Unable to load hunspell data! [dictionary=" + args.get("dictionary") + ",affix=" + affixFile + "]", e);
+ } finally {
+ IOUtils.closeWhileHandlingException(affix);
+ IOUtils.closeWhileHandlingException(dictionaries);
}
}
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/JapaneseTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/JapaneseTokenizerFactory.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/JapaneseTokenizerFactory.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/JapaneseTokenizerFactory.java Thu Jul 19 15:58:54 2012
@@ -42,9 +42,10 @@ import org.apache.lucene.analysis.util.R
* <fieldType name="text_ja" class="solr.TextField">
* <analyzer>
* <tokenizer class="solr.JapaneseTokenizerFactory"
- * mode=NORMAL
- * userDictionary=user.txt
- * userDictionaryEncoding=UTF-8
+ * mode="NORMAL"
+ * userDictionary="user.txt"
+ * userDictionaryEncoding="UTF-8"
+ * discardPunctuation="true"
* />
* <filter class="solr.JapaneseBaseFormFilterFactory"/>
* </analyzer>
@@ -58,9 +59,14 @@ public class JapaneseTokenizerFactory ex
private static final String USER_DICT_ENCODING = "userDictionaryEncoding";
+ private static final String DISCARD_PUNCTUATION = "discardPunctuation"; // Expert option
+
private UserDictionary userDictionary;
+
private Mode mode;
-
+
+ private boolean discardPunctuation;
+
@Override
public void inform(ResourceLoader loader) {
mode = getMode(args);
@@ -83,17 +89,18 @@ public class JapaneseTokenizerFactory ex
} catch (Exception e) {
throw new InitializationException("Exception thrown while loading dictionary", e);
}
+ discardPunctuation = getBoolean(DISCARD_PUNCTUATION, true);
}
@Override
public Tokenizer create(Reader input) {
- return new JapaneseTokenizer(input, userDictionary, true, mode);
+ return new JapaneseTokenizer(input, userDictionary, discardPunctuation, mode);
}
private Mode getMode(Map<String, String> args) {
String mode = args.get(MODE);
if (mode != null) {
- return Mode.valueOf(mode.toUpperCase(Locale.ENGLISH));
+ return Mode.valueOf(mode.toUpperCase(Locale.ROOT));
} else {
return JapaneseTokenizer.DEFAULT_MODE;
}
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilter.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilter.java Thu Jul 19 15:58:54 2012
@@ -21,12 +21,11 @@ import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
+import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.Set;
import org.apache.lucene.analysis.charfilter.BaseCharFilter;
-import org.apache.lucene.analysis.CharReader;
-import org.apache.lucene.analysis.CharStream;
/**
* <p>
@@ -71,21 +70,21 @@ public class LegacyHTMLStripCharFilter e
public static void main(String[] args) throws IOException {
Reader in = new LegacyHTMLStripCharFilter(
- CharReader.get(new InputStreamReader(System.in)));
+ new InputStreamReader(System.in, Charset.defaultCharset()));
int ch;
while ( (ch=in.read()) != -1 ) System.out.print((char)ch);
}
- public LegacyHTMLStripCharFilter(CharStream source) {
- super(source.markSupported() ? source : CharReader.get(new BufferedReader(source)));
+ public LegacyHTMLStripCharFilter(Reader source) {
+ super(source.markSupported() ? source : new BufferedReader(source));
}
- public LegacyHTMLStripCharFilter(CharStream source, Set<String> escapedTags){
+ public LegacyHTMLStripCharFilter(Reader source, Set<String> escapedTags){
this(source);
this.escapedTags = escapedTags;
}
- public LegacyHTMLStripCharFilter(CharStream source, Set<String> escapedTags, int readAheadLimit){
+ public LegacyHTMLStripCharFilter(Reader source, Set<String> escapedTags, int readAheadLimit){
this(source);
this.escapedTags = escapedTags;
this.readAheadLimit = readAheadLimit;
@@ -104,7 +103,7 @@ public class LegacyHTMLStripCharFilter e
return ch;
}
numRead++;
- return input.read();
+ return in.read();
}
private int nextSkipWS() throws IOException {
@@ -119,7 +118,7 @@ public class LegacyHTMLStripCharFilter e
return pushed.charAt(len-1);
}
numRead++;
- int ch = input.read();
+ int ch = in.read();
push(ch);
return ch;
}
@@ -181,11 +180,11 @@ public class LegacyHTMLStripCharFilter e
private void saveState() throws IOException {
lastMark = numRead;
- input.mark(readAheadLimit);
+ in.mark(readAheadLimit);
}
private void restoreState() throws IOException {
- input.reset();
+ in.reset();
pushed.setLength(0);
}
@@ -774,12 +773,6 @@ public class LegacyHTMLStripCharFilter e
return i;
}
- @Override
- public void close() throws IOException {
- input.close();
- }
-
-
private static final HashMap<String,Character> entityTable;
static {
entityTable = new HashMap<String,Character>();
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilterFactory.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilterFactory.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/LegacyHTMLStripCharFilterFactory.java Thu Jul 19 15:58:54 2012
@@ -18,7 +18,8 @@ package org.apache.solr.analysis;
* limitations under the License.
*/
-import org.apache.lucene.analysis.CharStream;
+import java.io.Reader;
+
import org.apache.lucene.analysis.util.CharFilterFactory;
/**
@@ -52,7 +53,7 @@ import org.apache.lucene.analysis.util.C
@Deprecated
public class LegacyHTMLStripCharFilterFactory extends CharFilterFactory {
- public LegacyHTMLStripCharFilter create(CharStream input) {
+ public LegacyHTMLStripCharFilter create(Reader input) {
return new LegacyHTMLStripCharFilter(input);
}
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/MappingCharFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/MappingCharFilterFactory.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/MappingCharFilterFactory.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/MappingCharFilterFactory.java Thu Jul 19 15:58:54 2012
@@ -19,12 +19,13 @@ package org.apache.solr.analysis;
import java.io.File;
import java.io.IOException;
+import java.io.Reader;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.lucene.analysis.CharStream;
+import org.apache.lucene.analysis.CharFilter;
import org.apache.lucene.analysis.charfilter.MappingCharFilter;
import org.apache.lucene.analysis.charfilter.NormalizeCharMap;
import org.apache.lucene.analysis.util.*;
@@ -78,7 +79,7 @@ public class MappingCharFilterFactory ex
}
}
- public CharStream create(CharStream input) {
+ public CharFilter create(Reader input) {
return new MappingCharFilter(normMap,input);
}
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/PatternReplaceCharFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/PatternReplaceCharFilterFactory.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/PatternReplaceCharFilterFactory.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/PatternReplaceCharFilterFactory.java Thu Jul 19 15:58:54 2012
@@ -17,10 +17,11 @@
package org.apache.solr.analysis;
+import java.io.Reader;
import java.util.Map;
import java.util.regex.Pattern;
-import org.apache.lucene.analysis.CharStream;
+import org.apache.lucene.analysis.CharFilter;
import org.apache.lucene.analysis.pattern.PatternReplaceCharFilter;
import org.apache.lucene.analysis.util.CharFilterFactory;
@@ -53,7 +54,7 @@ public class PatternReplaceCharFilterFac
// TODO: throw exception if you set maxBlockChars or blockDelimiters ?
}
- public CharStream create(CharStream input) {
+ public CharFilter create(Reader input) {
return new PatternReplaceCharFilter( p, replacement, input );
}
}
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/PersianCharFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/PersianCharFilterFactory.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/PersianCharFilterFactory.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/PersianCharFilterFactory.java Thu Jul 19 15:58:54 2012
@@ -17,7 +17,9 @@ package org.apache.solr.analysis;
* limitations under the License.
*/
-import org.apache.lucene.analysis.CharStream;
+import java.io.Reader;
+
+import org.apache.lucene.analysis.CharFilter;
import org.apache.lucene.analysis.fa.PersianCharFilter;
import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
import org.apache.lucene.analysis.util.CharFilterFactory;
@@ -37,7 +39,7 @@ import org.apache.lucene.analysis.util.M
public class PersianCharFilterFactory extends CharFilterFactory implements MultiTermAwareComponent {
@Override
- public CharStream create(CharStream input) {
+ public CharFilter create(Reader input) {
return new PersianCharFilter(input);
}
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/PhoneticFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/PhoneticFilterFactory.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/PhoneticFilterFactory.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/PhoneticFilterFactory.java Thu Jul 19 15:58:54 2012
@@ -68,12 +68,12 @@ public class PhoneticFilterFactory exten
private static final Map<String, Class<? extends Encoder>> registry = new HashMap<String, Class<? extends Encoder>>(6);
static {
- registry.put("DoubleMetaphone".toUpperCase(Locale.ENGLISH), DoubleMetaphone.class);
- registry.put("Metaphone".toUpperCase(Locale.ENGLISH), Metaphone.class);
- registry.put("Soundex".toUpperCase(Locale.ENGLISH), Soundex.class);
- registry.put("RefinedSoundex".toUpperCase(Locale.ENGLISH), RefinedSoundex.class);
- registry.put("Caverphone".toUpperCase(Locale.ENGLISH), Caverphone2.class);
- registry.put("ColognePhonetic".toUpperCase(Locale.ENGLISH), ColognePhonetic.class);
+ registry.put("DoubleMetaphone".toUpperCase(Locale.ROOT), DoubleMetaphone.class);
+ registry.put("Metaphone".toUpperCase(Locale.ROOT), Metaphone.class);
+ registry.put("Soundex".toUpperCase(Locale.ROOT), Soundex.class);
+ registry.put("RefinedSoundex".toUpperCase(Locale.ROOT), RefinedSoundex.class);
+ registry.put("Caverphone".toUpperCase(Locale.ROOT), Caverphone2.class);
+ registry.put("ColognePhonetic".toUpperCase(Locale.ROOT), ColognePhonetic.class);
}
protected boolean inject = true;
@@ -93,7 +93,7 @@ public class PhoneticFilterFactory exten
throw new InitializationException("Missing required parameter: " + ENCODER
+ " [" + registry.keySet() + "]");
}
- clazz = registry.get(name.toUpperCase(Locale.ENGLISH));
+ clazz = registry.get(name.toUpperCase(Locale.ROOT));
if( clazz == null ) {
clazz = resolveEncoder(name);
}
Modified: lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java?rev=1363400&r1=1363399&r2=1363400&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java (original)
+++ lucene/dev/branches/pforcodec_3892/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java Thu Jul 19 15:58:54 2012
@@ -22,17 +22,12 @@ import org.apache.lucene.analysis.util.C
import org.apache.lucene.analysis.util.TokenFilterFactory;
import org.apache.lucene.analysis.util.TokenizerFactory;
-import java.io.IOException;
import java.io.Reader;
/**
- *
+ * An analyzer that uses a tokenizer and a list of token filters to
+ * create a TokenStream.
*/
-
-//
-// An analyzer that uses a tokenizer and a list of token filters to
-// create a TokenStream.
-//
public final class TokenizerChain extends SolrAnalyzer {
final private CharFilterFactory[] charFilters;
final private TokenizerFactory tokenizer;
@@ -52,26 +47,10 @@ public final class TokenizerChain extend
public TokenizerFactory getTokenizerFactory() { return tokenizer; }
public TokenFilterFactory[] getTokenFilterFactories() { return filters; }
- class SolrTokenStreamComponents extends TokenStreamComponents {
- public SolrTokenStreamComponents(final Tokenizer source, final TokenStream result) {
- super(source, result);
- }
-
- // TODO: what is going on here?
- @Override
- protected void reset(Reader reader) throws IOException {
- // the tokenizers are currently reset by the indexing process, so only
- // the tokenizer needs to be reset.
- Reader r = initReader(null, reader);
- super.reset(r);
- }
- }
-
-
@Override
public Reader initReader(String fieldName, Reader reader) {
if (charFilters != null && charFilters.length > 0) {
- CharStream cs = CharReader.get( reader );
+ Reader cs = reader;
for (CharFilterFactory charFilter : charFilters) {
cs = charFilter.create(cs);
}
@@ -82,12 +61,12 @@ public final class TokenizerChain extend
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader aReader) {
- Tokenizer tk = tokenizer.create( initReader(fieldName, aReader) );
+ Tokenizer tk = tokenizer.create( aReader );
TokenStream ts = tk;
for (TokenFilterFactory filter : filters) {
ts = filter.create(ts);
}
- return new SolrTokenStreamComponents(tk, ts);
+ return new TokenStreamComponents(tk, ts);
}
@Override