You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2011/03/30 11:17:42 UTC
svn commit: r1086876 [13/18] - in /lucene/dev/branches/realtime_search: ./
dev-tools/eclipse/ dev-tools/idea/ dev-tools/idea/.idea/libraries/
dev-tools/idea/lucene/ dev-tools/idea/solr/
dev-tools/idea/solr/contrib/analysis-extras/ dev-tools/idea/solr/c...
Modified: lucene/dev/branches/realtime_search/solr/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/build.xml?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/build.xml (original)
+++ lucene/dev/branches/realtime_search/solr/build.xml Wed Mar 30 09:17:25 2011
@@ -30,7 +30,7 @@
<property name="src" location="src" />
<property name="web.xml" location="${src}/webapp/web/WEB-INF/web.xml"/>
- <!-- Destination for distribution files (demo WAR, src distro, etc.) -->
+ <!-- Destination for distribution files (demo WAR, contrib JARs, etc.) -->
<property name="dist" location="dist" />
<property name="clover.db.dir" location="${dest}/tests/clover/db"/>
@@ -57,6 +57,7 @@
<echo message="Use 'ant clean' to clean compiled files." />
<echo message="Use 'ant compile' to compile the source code." />
<echo message="Use 'ant dist' to build the project WAR and JAR files." />
+ <echo message="Use 'ant javadoc' to build javadoc under build/docs/api" />
<echo message="Use 'ant generate-maven-artifacts' to generate maven artifacts." />
<echo message="Use 'ant package' to generate zip, tgz for distribution." />
<echo message="Use 'ant luke' to start luke. see: http://luke.googlecode.com" />
@@ -68,12 +69,13 @@
description="Cleans compiled files and other temporary artifacts.">
<delete dir="${dest}" />
<delete dir="${dist}" />
+ <delete dir="${package.dir}" />
<delete dir="example/solr/lib" />
<delete dir="${lucene-libs}" />
</target>
<target name="clean-dest"
- description="Cleans out ${dest} but leaves ${dist} alone. This allows us to run nightly and clover together in Hudson">
+ description="Cleans out ${dest} but leaves ${dist} and ${package.dir} alone. This allows us to run nightly and clover together in Hudson">
<delete includeemptydirs="true" >
<fileset dir="${dest}">
<exclude name="docs/"/>
@@ -81,35 +83,6 @@
</delete>
</target>
- <!-- Prepares an entity file for use in Forrest documentation -->
- <target name="init-forrest-entities">
- <!-- no description, don't advertise -->
-
- <!-- Building off Lucene trunk, we don't need this
- <available file="lib/lucene-core-${version}.jar"
- property="lucene-version-ok-m2deploy-wont-fail" />
- <fail unless="lucene-version-ok-m2deploy-wont-fail">
- lib/lucene-core-${version}.jar Missing
-
- lib/lucene-core-${version}.jar does not exist.
- This will cause problems with m2-deploy later, so fail fast now.
-
- Probably cause: lucene jars were upgraded w/o modifying the
- 'version' property in common-build.xml
- </fail>
- -->
-
- <mkdir dir="${dest}" />
-
- <!-- A folder containing the expanded war file -->
- <mkdir dir="${dest}/web" />
-
- <echo file="${dest}/solr-specific-forrest-variables.ent" append="false">
- <!ENTITY solr.specversion "${specversion}">
- </echo>
- </target>
-
-
<!-- ========================================================================= -->
<!-- ===================== COMPILATION-RELATED TASKS ========================= -->
<!-- ========================================================================= -->
@@ -124,7 +97,7 @@
<target name="compile-solrj"
description="Compile the java client."
- depends="init-forrest-entities, compile-lucene">
+ depends="compile-lucene">
<solr-javac destdir="${dest}/solrj"
classpathref="compile.classpath">
@@ -147,7 +120,7 @@
<!-- Compile the project. -->
<target name="compile"
description="Compile the source code."
- depends="compile-solrj,init-forrest-entities">
+ depends="validate-solr, compile-solrj">
<solr-javac destdir="${dest}/solr"
classpathref="compile.classpath.solrj">
@@ -156,7 +129,13 @@
</solr-javac>
</target>
- <target name="javadoc-solrj" depends="compile-solrj" description="Generates solrj javadoc documentation.">
+ <target name="javadoc-dep">
+ <copy failonerror="false" todir="${build.docs}">
+ <fileset dir="site" />
+ </copy>
+ </target>
+
+ <target name="javadoc-solrj" depends="compile-solrj,javadoc-dep" description="Generates solrj javadoc documentation.">
<sequential>
<mkdir dir="${build.javadoc}/solrj"/>
@@ -175,7 +154,7 @@
</sequential>
</target>
- <target name="javadoc-core" depends="compile" description="Generates javadoc documentation for core.">
+ <target name="javadoc-core" depends="compile,javadoc-dep" description="Generates javadoc documentation for core.">
<sequential>
<mkdir dir="${build.javadoc}/solr"/>
@@ -198,7 +177,7 @@
</target>
- <target name="javadoc-all" description="Generate javadoc for core, client and contrib">
+ <target name="javadoc-all" depends="compile,javadoc-dep" description="Generate javadoc for core, client and contrib">
<sequential>
<mkdir dir="${build.javadoc}"/>
@@ -207,6 +186,7 @@
<path refid="compile.classpath.solrj"/>
<fileset dir="contrib">
<include name="**/lib/**/*.jar"/>
+ <include name="**/lucene-libs/**/*.jar"/>
</fileset>
<pathelement location="${dest}/client/solrj"/>
</path>
@@ -218,6 +198,7 @@
<packageset dir="${src}/java" />
<packageset dir="${src}/webapp/src" />
<packageset dir="contrib/dataimporthandler/src/main/java" />
+ <packageset dir="contrib/dataimporthandler/src/extras/main/java" />
<packageset dir="contrib/clustering/src/main/java" />
<packageset dir="contrib/extraction/src/main/java" />
<packageset dir="contrib/uima/src/main/java" />
@@ -234,7 +215,7 @@
</sequential>
</target>
- <target name="javadoc-contrib" description="Generate javadoc for contrib classes" depends="build-contrib">
+ <target name="javadoc-contrib" description="Generate javadoc for contrib classes" depends="build-contrib,javadoc-dep">
<contrib-crawl target="javadoc"
failonerror="true"/>
</target>
@@ -374,12 +355,12 @@
<!-- Run core unit tests. -->
<target name="test-core"
description="Runs the core unit tests."
- depends="init-forrest-entities, compileTests, junit" />
+ depends="compileTests, junit" />
<!-- Run contrib unit tests. -->
<target name="test"
description="Runs the core unit tests."
- depends="test-core, test-contrib" />
+ depends="test-core, test-contrib, test-jsp" />
<target name="junit" depends="compileTests,junit-mkdir,junit-sequential,junit-parallel"/>
@@ -554,6 +535,37 @@
</clover-report>
</target>
+ <!-- ========================================================================= -->
+
+ <!-- Checks that all JSP files in the webapp compile successfully using Jetty's Jasper -->
+ <target name="test-jsp" depends="compile">
+ <property name="jsp.target" location="${dest}/jsp-temp" />
+ <taskdef classname="org.apache.jasper.JspC" name="jasper" >
+ <classpath>
+ <fileset dir="example/lib" includes="**/*.jar" />
+ </classpath>
+ </taskdef>
+ <delete dir="${jsp.target}" />
+ <mkdir dir="${jsp.target}" />
+ <jasper
+ uriroot="${src}/webapp/web"
+ outputDir="${jsp.target}"
+ compile="false"
+ verbose="1"
+ package="j"
+ />
+ <javac
+ srcdir="${jsp.target}"
+ destdir="${jsp.target}"
+ target="${java.compat.version}"
+ source="${java.compat.version}"
+ debug="off"
+ encoding="utf8"
+ includeAntRuntime="${javac.includeAntRuntime}"
+ classpathref="test.compile.classpath"
+ />
+ </target>
+
<!-- ========================================================================= -->
<!-- ===================== DISTRIBUTION-RELATED TASKS ======================== -->
@@ -563,12 +575,12 @@
<!-- Creates the Solr distribution files. -->
<target name="dist"
description="Creates the Solr distribution files."
- depends="init-forrest-entities, dist-solrj, dist-jar, dist-contrib, dist-war" />
+ depends="dist-solrj, dist-jar, dist-contrib, dist-war" />
<!-- Creates the Solr WAR file. -->
<target name="dist-war"
description="Creates the Solr WAR Distribution file."
- depends="compile, make-manifest, dist-jar, dist-solrj, lucene-jars-to-solr">
+ depends="compile, test-jsp, make-manifest, dist-jar, dist-solrj, lucene-jars-to-solr">
<mkdir dir="${dist}" />
<war destfile="${dist}/${fullnamever}.war"
webxml="${web.xml}"
@@ -576,7 +588,7 @@
manifest="${dest}/META-INF/MANIFEST.MF">
<lib dir="${lib}">
<exclude name="servlet-api*.jar" />
- <exclude name="easymock.jar" />
+ <exclude name="easymock-*.jar" />
<exclude name="junit-*.jar" />
<exclude name="*.txt" />
<exclude name="*.template" />
@@ -597,7 +609,7 @@
</war>
</target>
- <target name="dist-src" description="Creates the Solr source distribution files"
+ <target name="dist-src" description="Creates the Solr source distribution files for maven"
depends="make-manifest">
<mkdir dir="${dist}" />
@@ -628,7 +640,7 @@
</solr-jar>
</target>
- <target name="dist-javadoc" description="Creates the Solr javadoc distribution files"
+ <target name="dist-javadoc" description="Creates the Solr javadoc distribution files for maven"
depends="make-manifest, javadoc">
<mkdir dir="${dist}" />
@@ -685,7 +697,7 @@
<target name="example"
description="Creates a runnable example configuration."
- depends="init-forrest-entities,compile-lucene,dist-contrib,dist-war,example-contrib">
+ depends="compile-lucene,dist-contrib,dist-war,example-contrib">
<copy file="${dist}/${fullnamever}.war"
tofile="${example}/webapps/${ant.project.name}.war"/>
<jar destfile="${example}/exampledocs/post.jar"
@@ -725,55 +737,120 @@
</target>
<!-- make a distribution -->
- <target name="package" depends="create-package"/>
+ <target name="package" depends="package-src,create-package"/>
+
+ <property name="svn.export.dir" location="${dest}/svn-export"/>
+
+ <!-- Makes a tarball from running "svn export" at the root level. -->
+ <!-- Copies NOTICE.txt and LICENSE.txt from solr/ to the root level. -->
+ <target name="package-src" depends="svn-export-source"
+ description="Packages the Solr Source Distribution">
+ <mkdir dir="${package.dir}"/>
+ <property name="source.package.file"
+ value="${package.dir}/${fullnamever}-src.tgz"/>
+ <delete file="${source.package.file}" failonerror="false" />
+ <tar destfile="${source.package.file}" compression="gzip" longfile="gnu">
+ <tarfileset dir="${svn.export.dir}" prefix="${fullnamever}"/>
+ <tarfileset dir="${svn.export.dir}/solr" prefix="${fullnamever}"
+ includes="NOTICE.txt,LICENSE.txt"/>
+ </tar>
+ <solr-checksum file="${source.package.file}"/>
+ </target>
+
+ <!-- Runs "svn export" in build/svn-export/ with the same root-level URL -->
+ <!-- and revision as the current working copy. -->
+ <target name="svn-export-source" depends="get-svn-info">
+ <delete dir="${svn.export.dir}" includeemptydirs="true" failonerror="false"/>
+ <mkdir dir="${dest}"/>
+ <exec dir="." executable="${svn.exe}" failonerror="true">
+ <arg value="export"/>
+ <arg value="--native-eol"/>
+ <arg value="LF"/>
+ <arg value="-r"/>
+ <arg value="${svn.Revision}"/>
+ <arg value="${svn.URL}"/>
+ <arg value="${svn.export.dir}"/>
+ </exec>
+ </target>
+
+ <!-- Populates properties svn.URL and svn.Revision using "svn info" -->
+ <target name="get-svn-info">
+ <exec dir=".." executable="${svn.exe}" outputproperty="svn.info" failonerror="true">
+ <arg value="info"/>
+ </exec>
+ <loadproperties>
+ <propertyresource name="svn.info"/>
+ <filterchain>
+ <linecontainsregexp>
+ <regexp pattern="(URL|Revision):"/>
+ </linecontainsregexp>
+ <replacestring from=": " to="="/>
+ <prefixlines prefix="svn."/>
+ </filterchain>
+ </loadproperties>
+ </target>
<target name="create-package"
- description="Packages the Solr Distribution files and Documentation."
- depends="dist, example, javadoc, dist-src, dist-javadoc">
+ description="Packages the Solr Binary Distribution"
+ depends="dist, example, javadoc">
+ <mkdir dir="${package.dir}"/>
- <copy failonerror="false" todir="${build.docs}">
- <fileset dir="site" />
- </copy>
+ <delete includeemptydirs="true">
+ <fileset dir="${example}/work" includes="**/*"/>
+ </delete>
- <delete file="${dist}/${fullnamever}.tgz" failonerror="false" />
- <delete file="${dist}/${fullnamever}.zip" failonerror="false" />
+ <delete includeemptydirs="true" failonerror="false">
+ <fileset dir="${dest}/${fullnamever}" includes="**/*"/>
+ </delete>
- <tar destfile="${dist}/${fullnamever}.tgz" compression="gzip" longfile="gnu">
- <tarfileset dir="."
- prefix="${fullnamever}"
- includes="LICENSE.txt NOTICE.txt *.txt *.xml lucene-libs/** lib/** src/** example/** client/** contrib/"
- excludes="lib/README.committers.txt **/data/ **/logs/* **/classes/ **/*.sh **/bin/ src/scripts/ src/site/build/ **/target/ client/ruby/flare/ client/python contrib/**/build/ **/*.iml **/*.ipr **/*.iws contrib/clustering/example/lib/** contrib/clustering/lib/downloads/** contrib/analysis-extras/lib/** contrib/uima/lib/**" />
+ <delete file="${package.dir}/${fullnamever}.tgz" failonerror="false" />
+ <delete file="${package.dir}/${fullnamever}.zip" failonerror="false" />
+
+ <tar destfile="${package.dir}/${fullnamever}.tgz" compression="gzip" longfile="gnu">
<tarfileset dir="."
prefix="${fullnamever}"
- includes="src/test-files/solr/lib/classes/empty-file-main-lib.txt" />
+ includes="LICENSE.txt NOTICE.txt *.txt example/** client/** contrib/**/lib/ contrib/**/lucene-libs/ contrib/**/README.txt contrib/**/CHANGES.txt"
+ excludes="${dist}/** ${dest}/** lib/README.committers.txt **/data/ **/logs/* **/classes/ **/*.sh **/bin/ src/scripts/** src/site/build/** **/target/** client/ruby/flare/** client/python/** client/javascript/** contrib/**/build/** **/*.iml **/*.ipr **/*.iws **/*pom.xml.template" />
<tarfileset dir="."
mode="755"
prefix="${fullnamever}"
- includes="**/*.sh **/bin/ src/scripts/" />
+ includes="example/**/*.sh example/**/bin/" />
<tarfileset dir="."
prefix="${fullnamever}"
- includes="dist/**"
- excludes="*.tgz *.zip *.md5 **/*src*.jar **/*docs*.jar" />
+ includes="dist/*.jar dist/*.war dist/solrj-lib/*"
+ excludes="**/*.tgz **/*.zip **/*.md5 **/*src*.jar **/*docs*.jar" />
+ <!-- hack: the javadocs are built twice since maven needs separate packages... exclude those -->
<tarfileset dir="${build.docs}"
- prefix="${fullnamever}/docs/" />
- <tarfileset dir="../lucene"
- prefix="lucene"
- excludes="**/build/" />
- <tarfileset dir="../modules"
- prefix="modules"
- excludes="**/build/" />
+ prefix="${fullnamever}/docs/"
+ excludes="api/solr/** api/contrib-*/**"
+ />
</tar>
- <solr-checksum file="${dist}/${fullnamever}.tgz"/>
+ <solr-checksum file="${package.dir}/${fullnamever}.tgz"/>
- <gunzip src="${dist}/${fullnamever}.tgz" dest="${dest}/${fullnamever}.tar"/>
+ <gunzip src="${package.dir}/${fullnamever}.tgz" dest="${dest}/${fullnamever}.tar"/>
<untar src="${dest}/${fullnamever}.tar" dest="${dest}"/>
+ <!--
+ This is a list of text file patterns to convert to CRLF line-ending style.
+ Shell scripts and files included in shell scripts should not be converted.
+ NB: The line-ending conversion process will mangle non-UTF8-encoded files.
+ -->
<fixcrlf srcdir="${dest}/${fullnamever}"
+ encoding="UTF-8"
eol="crlf"
- includes="**/*.txt **/*.xml **/*.java **/*.html **/*.csv **/*.css **/*.properties **/*.jsp **/*.xsl **/*.py **/*.rb **/*.js **/*.json **/*.pl"
+ includes="**/*.alg **/*.cfg **/*.cgi **/*.cpp **/*.css **/*.csv **/*.dtd
+ **/*.erb **/*.fcgi **/.htaccess **/*.htm **/*.html **/*.incl
+ **/*.java **/*.javacc **/*.jflex **/*.jflex-macro **/*.jj
+ **/*.js **/*.json **/*.jsp **/*LICENSE **/package-list **/*.pl
+ **/*.pom **/*pom.xml.template **/*.properties **/*.py
+ **/*.rake **/Rakefile **/*.rb **/*.rbbi **/README* **/*.rhtml
+ **/*.rslp **/*.rxml **/*.script **/*.svg **/*.tsv **/*.txt
+ **/UPGRADING **/USAGE **/*.uxf **/*.vm **/*.xcat **/*.xml
+ **/*.xsl **/*.xslt **/*.yml"
+ excludes="**/stopwordsWrongEncoding.txt **/gb18030-example.xml"
/>
- <zip destfile="${dist}/${fullnamever}.zip">
+ <zip destfile="${package.dir}/${fullnamever}.zip">
<zipfileset dir="${dest}/${fullnamever}"
prefix="${fullnamever}"
excludes="**/*.sh **/bin/ src/scripts/" />
@@ -781,60 +858,51 @@
prefix="${fullnamever}"
includes="**/*.sh **/bin/ src/scripts/"
filemode="755" />
- <zipfileset dir="../lucene"
- prefix="lucene"
- excludes="**/build/" />
- <zipfileset dir="../modules"
- prefix="modules"
- excludes="**/build/" />
</zip>
- <solr-checksum file="${dist}/${fullnamever}.zip"/>
+ <solr-checksum file="${package.dir}/${fullnamever}.zip"/>
</target>
- <target name="build-site" depends="svn-up, init-forrest-entities" description="Prototype Helper for Committers. Assumes SVN is in the path">
- <delete dir="src/site/build"/>
- <exec executable="forrest" dir="src/site"/>
- <copy todir="site">
- <fileset dir="src/site/build/site"/>
- </copy>
- <exec executable="svn" dir="site">
- <arg value="ci"/>
- <arg value="-m"/>
- <arg value=""Forrest Entity updates""/>
- </exec>
- <exec executable="svn" dir="src/site">
- <arg value="ci"/>
- <arg value="-m"/>
- <arg value=""Forrest Entity updates""/>
- </exec>
- <antcall target="svn-up"/>
+ <target name="build-site" depends="svn-up"
+ description="Prototype Helper for Committers. Assumes SVN is in the path">
+ <delete dir="src/site/build"/>
+ <exec executable="forrest" dir="src/site"/>
+ <copy todir="site">
+ <fileset dir="src/site/build/site"/>
+ </copy>
+ <antcall target="svn-up"/>
+
+ </target>
+ <target name="svn-up">
+ <exec executable="${svn.exe}">
+ <arg value="update"/>
+ </exec>
</target>
- <target name="svn-up">
- <exec executable="svn">
- <arg value="update"/>
- </exec>
- </target>
- <target name="clean-dist-signatures">
+ <target name="clean-package-signatures">
<delete failonerror="false">
- <fileset dir="${dist}">
+ <fileset dir="${package.dir}">
<include name="**/*.asc"/>
</fileset>
</delete>
</target>
- <target name="sign-artifacts" depends="clean-dist-signatures">
+ <target name="sign-artifacts" depends="clean-package-signatures">
<!--<property file="${user.home}/.solr/build.properties" />-->
<input message="password:>" addproperty="gpg.passphrase">
<handler classname="org.apache.tools.ant.input.SecureInputHandler" />
</input>
- <sign-artifact input.file="${dist}/${fullnamever}.tgz" output.file="${dist}/${fullnamever}.tgz.asc" gpg.passphrase="${gpg.passphrase}"/>
- <sign-artifact input.file="${dist}/${fullnamever}.zip" output.file="${dist}/${fullnamever}.zip.asc" gpg.passphrase="${gpg.passphrase}"/>
+ <sign-artifact input.file="${package.dir}/${fullnamever}.tgz" output.file="${package.dir}/${fullnamever}.tgz.asc" gpg.passphrase="${gpg.passphrase}"/>
+ <sign-artifact input.file="${package.dir}/${fullnamever}.zip" output.file="${package.dir}/${fullnamever}.zip.asc" gpg.passphrase="${gpg.passphrase}"/>
+ <sign-artifact input.file="${package.dir}/${fullnamever}-src.tgz" output.file="${package.dir}/${fullnamever}-src.tgz.asc" gpg.passphrase="${gpg.passphrase}"/>
<!-- Maven artifacts -->
<sign-maven-dependency-artifacts artifact.id="solr-commons-csv" gpg.passphrase="${gpg.passphrase}"/>
<sign-maven-dependency-artifacts artifact.id="solr-noggit" gpg.passphrase="${gpg.passphrase}"/>
+ <sign-maven-dependency-artifacts artifact.id="solr-uima-an-alchemy" gpg.passphrase="${gpg.passphrase}"/>
+ <sign-maven-dependency-artifacts artifact.id="solr-uima-an-calais" gpg.passphrase="${gpg.passphrase}"/>
+ <sign-maven-dependency-artifacts artifact.id="solr-uima-an-tagger" gpg.passphrase="${gpg.passphrase}"/>
+ <sign-maven-dependency-artifacts artifact.id="solr-uima-an-wst" gpg.passphrase="${gpg.passphrase}"/>
<sign-maven-war-artifacts artifact.id="solr" gpg.passphrase="${gpg.passphrase}"/>
<sign-maven-artifacts artifact.id="solr-analysis-extras" gpg.passphrase="${gpg.passphrase}"/>
<sign-maven-artifacts artifact.id="solr-cell" gpg.passphrase="${gpg.passphrase}"/>
@@ -843,22 +911,42 @@
<sign-maven-artifacts artifact.id="solr-dataimporthandler" gpg.passphrase="${gpg.passphrase}"/>
<sign-maven-artifacts artifact.id="solr-dataimporthandler-extras" gpg.passphrase="${gpg.passphrase}"/>
<sign-maven-artifacts artifact.id="solr-solrj" gpg.passphrase="${gpg.passphrase}"/>
+ <sign-maven-artifacts artifact.id="solr-uima" gpg.passphrase="${gpg.passphrase}"/>
<!-- These are special since there are no jars, just poms -->
<sign-artifact input.file="${maven.dist.prefix}/solr-parent/${version}/solr-parent-${version}.pom" gpg.passphrase="${gpg.passphrase}"/>
</target>
- <target name="prepare-release" depends="clean, svn-up, build-site, package, sign-artifacts" description="Prototype helper for Committers. Assumes gpg is in the path">
-
- <tar destfile="${dist}/solr-maven.tar" longfile="gnu">
- <tarfileset dir="${dist}/maven" prefix="maven"/>
- </tar>
- <tar destfile="${dist}/solr.tar" longfile="gnu">
- <tarfileset dir="${dist}" includes="${fullnamever}.* solr-maven.tar" excludes="${fullnamever}.war"/>
- </tar>
+ <property name="rc" value="rc0"/>
+ <property name="remote.staging.dir" value="public_html/staging_area/${rc}/${version}"/>
+ <property name="keyfile" value="${user.home}/.ssh/id_rsa"/>
+ <property name="scp.user" value="${user.name}"/>
+ <!--keys.dir is the location of the https://svn.apache.org/repos/asf/lucene/java/dist/ directory-->
+ <property name="keys.dir" value="${common-solr.dir}/../../dist"/>
+ <target name="copy-to-stage">
+ <sshexec host="people.apache.org"
+ username="${scp.user}"
+ keyfile="${keyfile}"
+ command="mkdir -p ${remote.staging.dir}"/>
+ <echo>Uploading artifacts to ${scp.user}@people.apache.org:${remote.staging.dir}</echo>
+ <scp todir="${scp.user}@people.apache.org:${remote.staging.dir}"
+ username="${scp.user}"
+ keyfile="${keyfile}"
+ verbose="true"
+ >
+ <fileset dir="dist"/>
+ <fileset dir="${keys.dir}">
+ <include name="KEYS"/>
+ </fileset>
+ </scp>
</target>
- <target name="generate-maven-artifacts" depends="maven.ant.tasks-check, create-package">
+ <target name="prepare-release" depends="clean, svn-up, package, sign-artifacts"
+ description="Prototype helper for Committers. Assumes gpg is in the path"/>
+
+ <target name="stage" depends="prepare-release, copy-to-stage"/>
+
+ <target name="generate-maven-artifacts" depends="maven.ant.tasks-check,dist,dist-src,dist-javadoc">
<sequential>
<ant target="get-maven-poms" dir=".."/>
@@ -877,14 +965,26 @@
<m2-deploy pom.xml="pom.xml"/>
- <!-- ========== SOLR SPECIFIC COMMONS CSV ========== -->
+ <!-- ========== SOLR SPECIFIC NON-MAVENIZED DEPENDENCIES ========== -->
<m2-deploy-with-pom-template pom.xml="lib/solr-commons-csv-pom.xml.template"
jar.file="lib/commons-csv-1.0-SNAPSHOT-r966014.jar" />
- <!-- ========== SOLR ARTIFACTS ========== -->
-
<m2-deploy-with-pom-template pom.xml="lib/apache-solr-noggit-pom.xml.template"
jar.file="lib/apache-solr-noggit-r944541.jar" />
+
+ <m2-deploy-with-pom-template pom.xml="contrib/uima/lib/solr-uima-an-alchemy-pom.xml.template"
+ jar.file="contrib/uima/lib/uima-an-alchemy-2.3.1-SNAPSHOT-r1062868.jar" />
+
+ <m2-deploy-with-pom-template pom.xml="contrib/uima/lib/solr-uima-an-calais-pom.xml.template"
+ jar.file="contrib/uima/lib/uima-an-calais-2.3.1-SNAPSHOT-r1062868.jar" />
+
+ <m2-deploy-with-pom-template pom.xml="contrib/uima/lib/solr-uima-an-tagger-pom.xml.template"
+ jar.file="contrib/uima/lib/uima-an-tagger-2.3.1-SNAPSHOT-r1062868.jar" />
+
+ <m2-deploy-with-pom-template pom.xml="contrib/uima/lib/solr-uima-an-wst-pom.xml.template"
+ jar.file="contrib/uima/lib/uima-an-wst-2.3.1-SNAPSHOT-r1076132.jar" />
+
+ <!-- ========== SOLR ARTIFACTS ========== -->
<m2-deploy pom.xml="contrib/dataimporthandler/src/pom.xml"
jar.file="${dist}/apache-solr-dataimporthandler-${version}.jar">
Modified: lucene/dev/branches/realtime_search/solr/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/common-build.xml?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/common-build.xml (original)
+++ lucene/dev/branches/realtime_search/solr/common-build.xml Wed Mar 30 09:17:25 2011
@@ -22,6 +22,7 @@
</description>
<dirname file="${ant.file.common-solr}" property="common-solr.dir"/>
+ <import file="${common-solr.dir}/../common-build.xml"/>
<!-- change this together with the default and test's solrconfig.xml after starting a new development branch: -->
<property name="tests.luceneMatchVersion" value="4.0"/>
@@ -34,7 +35,8 @@
<property name="name" value="${ant.project.name}"/>
<property name="dev-tools.dir" value="${solr-path}/../dev-tools"/>
- <property name="prettify.dir" value="${dev-tools.dir}/prettify"/>
+ <property name="prettify.dir" value="${solr-path}/../lucene/src/tools/prettify"/>
+ <property name="package.dir" location="package"/>
<tstamp>
<format property="year" pattern="yyyy"/>
@@ -80,6 +82,7 @@
default PATH. (this is useful for Hudson)
-->
<property name="svnversion.exe" value="svnversion" />
+ <property name="svn.exe" value="svn" />
<!-- Java Version we are compatible with -->
<property name="java.compat.version" value="1.6" />
@@ -150,7 +153,7 @@
<!-- Maven properties -->
<property name="maven.build.dir" value="${basedir}/build/maven"/>
- <property name="maven.dist.dir" value="${basedir}/dist/maven"/>
+ <property name="maven.dist.dir" value="${package.dir}/maven"/>
<property name="maven.dist.prefix" value="${maven.dist.dir}/org/apache/solr"/>
@@ -306,25 +309,10 @@
-->
<macrodef name="solr-checksum">
<attribute name="file"/>
- <!-- NOTE: we use the value of @{file} in the names any properties
- set because macro's don't have variables, and otherwise we
- wouldn't be able to checksum more then one file per build
- -->
- <sequential>
- <checksum file="@{file}"
- property="@{file}.sum"
- algorithm="${checksum.algorithm}" />
- <basename file="@{file}" property="@{file}.base" />
- <concat destfile="@{file}.${checksum.algorithm}"
- force="yes"
- append="false"
- fixlastline="yes">
- <header trimleading="yes">${@{file}.sum} </header>
- <!-- empty fileset to trick concat -->
- <fileset dir="." excludes="**" />
- <footer trimleading="yes">${@{file}.base}
- </footer>
- </concat>
+ <sequential>
+ <echo>Building checksums for '@{file}'</echo>
+ <checksum file="@{file}" algorithm="md5" format="MD5SUM" forceoverwrite="yes" readbuffersize="65536"/>
+ <checksum file="@{file}" algorithm="sha1" format="MD5SUM" forceoverwrite="yes" readbuffersize="65536"/>
</sequential>
</macrodef>
@@ -455,34 +443,32 @@
<macrodef name="sign-maven-artifacts" description="Signs maven artifacts">
<attribute name="artifact.id"/>
<attribute name="prefix.dir" default="${maven.dist.prefix}"/>
- <attribute name="maven.version" default="${version}"/>
<attribute name="gpg.passphrase"/>
<sequential>
- <sign-artifact input.file="@{prefix.dir}/@{artifact.id}/@{maven.version}/@{artifact.id}-@{maven.version}.jar" gpg.passphrase="@{gpg.passphrase}"/>
- <sign-artifact input.file="@{prefix.dir}/@{artifact.id}/@{maven.version}/@{artifact.id}-@{maven.version}-javadoc.jar" gpg.passphrase="@{gpg.passphrase}"/>
- <sign-artifact input.file="@{prefix.dir}/@{artifact.id}/@{maven.version}/@{artifact.id}-@{maven.version}-sources.jar" gpg.passphrase="@{gpg.passphrase}"/>
- <sign-artifact input.file="@{prefix.dir}/@{artifact.id}/@{maven.version}/@{artifact.id}-@{maven.version}.pom" gpg.passphrase="@{gpg.passphrase}"/>
+ <sign-artifact input.file="@{prefix.dir}/@{artifact.id}/${version}/@{artifact.id}-${version}.jar" gpg.passphrase="@{gpg.passphrase}"/>
+ <sign-artifact input.file="@{prefix.dir}/@{artifact.id}/${version}/@{artifact.id}-${version}-javadoc.jar" gpg.passphrase="@{gpg.passphrase}"/>
+ <sign-artifact input.file="@{prefix.dir}/@{artifact.id}/${version}/@{artifact.id}-${version}-sources.jar" gpg.passphrase="@{gpg.passphrase}"/>
+ <sign-artifact input.file="@{prefix.dir}/@{artifact.id}/${version}/@{artifact.id}-${version}.pom" gpg.passphrase="@{gpg.passphrase}"/>
</sequential>
</macrodef>
<macrodef name="sign-maven-war-artifacts" description="Signs maven artifacts">
<attribute name="artifact.id"/>
<attribute name="prefix.dir" default="${maven.dist.prefix}"/>
- <attribute name="maven.version" default="${version}"/>
<attribute name="gpg.passphrase"/>
<sequential>
- <sign-artifact input.file="@{prefix.dir}/@{artifact.id}/@{maven.version}/@{artifact.id}-@{maven.version}.war" gpg.passphrase="@{gpg.passphrase}"/>
- <sign-artifact input.file="@{prefix.dir}/@{artifact.id}/@{maven.version}/@{artifact.id}-@{maven.version}.pom" gpg.passphrase="@{gpg.passphrase}"/>
+ <sign-artifact input.file="@{prefix.dir}/@{artifact.id}/${version}/@{artifact.id}-${version}.war" gpg.passphrase="@{gpg.passphrase}"/>
+ <sign-artifact input.file="@{prefix.dir}/@{artifact.id}/${version}/@{artifact.id}-${version}.pom" gpg.passphrase="@{gpg.passphrase}"/>
</sequential>
</macrodef>
- <macrodef name="sign-maven-dependency-artifacts" description="Signs maven artifacts">
+ <macrodef name="sign-maven-dependency-artifacts" description="Signs a maven artifact and its POM">
<attribute name="artifact.id"/>
<attribute name="prefix.dir" default="${maven.dist.prefix}"/>
- <attribute name="maven.version" default="${version}"/>
<attribute name="gpg.passphrase"/>
<sequential>
- <sign-artifact input.file="@{prefix.dir}/@{artifact.id}/@{maven.version}/@{artifact.id}-@{maven.version}.jar" gpg.passphrase="@{gpg.passphrase}"/>
+ <sign-artifact input.file="@{prefix.dir}/@{artifact.id}/${version}/@{artifact.id}-${version}.jar" gpg.passphrase="@{gpg.passphrase}"/>
+ <sign-artifact input.file="@{prefix.dir}/@{artifact.id}/${version}/@{artifact.id}-${version}.pom" gpg.passphrase="@{gpg.passphrase}"/>
</sequential>
</macrodef>
@@ -576,13 +562,52 @@
</target>
<target name="maven.ant.tasks-check">
- <fail unless="maven.ant.tasks.present">
- ##################################################################
+ <fail unless="maven.ant.tasks.present">#
+ ##########################################################################
Maven ant tasks not found.
- Please make sure the maven-ant-tasks jar is in ANT_HOME/lib, or made
- available to Ant using other mechanisms like -lib or CLASSPATH.
- ##################################################################
+
+ Please download the Maven ant tasks JAR (maven-ant-tasks-2.1.1.jar)
+ from http://maven.apache.org/ant-tasks/download.html and add it to your
+ $$HOME/.ant/lib/ directory, or to your $$ANT_HOME/lib/ directory, or
+ to your $$CLASSPATH, or add "-lib /path/to/maven-ant-tasks-2.1.1.jar"
+ to the ant command.
+ ##########################################################################
</fail>
</target>
+ <!-- Validation -->
+ <target name="validate" depends="validate-solr"/>
+ <target name="validate-solr" depends="check-legal-solr" unless="validated-solr"/>
+
+ <target name="check-legal-solr" depends="compile-tools">
+ <java classname="org.apache.lucene.validation.DependencyChecker" failonerror="true" fork="true">
+ <classpath>
+ <path refid="tools.runtime.classpath" />
+ </classpath>
+ <!-- TODO: it might be better to just automatically find all directories that contain jar files, but that could take a
+ long time. This should be faster, but we could miss a directory
+ -->
+ <!-- Solr -->
+ <arg value="-c" />
+ <arg value="${basedir}/lib" />
+ <arg value="-c" />
+ <arg value="${basedir}/contrib/analysis-extras/lib" />
+ <arg value="-c" />
+ <arg value="${basedir}/contrib/clustering/lib" />
+ <arg value="-c" />
+ <arg value="${basedir}/contrib/dataimporthandler/lib" />
+ <arg value="-c" />
+ <arg value="${basedir}/contrib/extraction/lib" />
+ <arg value="-c" />
+ <arg value="${basedir}/contrib/uima/lib" />
+ <arg value="-c" />
+ <arg value="${basedir}/example/example-DIH/solr/db/lib" />
+ <arg value="-c" />
+ <arg value="${basedir}/example/example-DIH/solr/mail/lib" />
+ <arg value="-c" />
+ <arg value="${basedir}/example/example/lib" />
+ <arg value="-c" />
+ <arg value="${basedir}/src/test-files/solr/lib" />
+ </java>
+ </target>
</project>
Modified: lucene/dev/branches/realtime_search/solr/contrib/analysis-extras/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/contrib/analysis-extras/CHANGES.txt?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/contrib/analysis-extras/CHANGES.txt (original)
+++ lucene/dev/branches/realtime_search/solr/contrib/analysis-extras/CHANGES.txt Wed Mar 30 09:17:25 2011
@@ -15,6 +15,10 @@ $Id$
(No Changes)
+================== Release 3.2-dev ==================
+
+(No Changes)
+
================== Release 3.1-dev ==================
* SOLR-2210: Add icu-based tokenizer and filters to contrib/analysis-extras (rmuir)
Modified: lucene/dev/branches/realtime_search/solr/contrib/clustering/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/contrib/clustering/CHANGES.txt?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/contrib/clustering/CHANGES.txt (original)
+++ lucene/dev/branches/realtime_search/solr/contrib/clustering/CHANGES.txt Wed Mar 30 09:17:25 2011
@@ -11,6 +11,10 @@ $Id$
(No Changes)
+================== Release 3.2.0-dev ==================
+
+(No Changes)
+
================== Release 3.1.0-dev ==================
* SOLR-1684: Switch to use the SolrIndexSearcher.doc(int, Set<String>) method b/c it can use the document cache (gsingers)
Modified: lucene/dev/branches/realtime_search/solr/contrib/clustering/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/contrib/clustering/build.xml?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/contrib/clustering/build.xml (original)
+++ lucene/dev/branches/realtime_search/solr/contrib/clustering/build.xml Wed Mar 30 09:17:25 2011
@@ -30,7 +30,7 @@
<property name="example.local" value="example"/>
<path id="common.classpath">
- <fileset dir="lib"/>
+ <fileset dir="lib" includes="*.jar"/>
<pathelement location="${solr-path}/build/solr"/>
<pathelement location="${solr-path}/build/solrj"/>
<path refid="lucene.classpath"/>
Modified: lucene/dev/branches/realtime_search/solr/contrib/dataimporthandler/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/contrib/dataimporthandler/CHANGES.txt?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/contrib/dataimporthandler/CHANGES.txt (original)
+++ lucene/dev/branches/realtime_search/solr/contrib/dataimporthandler/CHANGES.txt Wed Mar 30 09:17:25 2011
@@ -12,6 +12,10 @@ $Id$
(No Changes)
+================== 3.2.0-dev ==================
+
+(No Changes)
+
================== 3.1.0-dev ==================
Upgrading from Solr 1.4
----------------------
@@ -72,6 +76,18 @@ Bug Fixes
* SOLR-1811: formatDate should use the current NOW value always (Sean Timm via noble)
+* SOLR-1794: Dataimport of CLOB fields fails when getCharacterStream() is
+ defined in a superclass. (Gunnar Gauslaa Bergem via rmuir)
+
+* SOLR-2057: DataImportHandler never calls UpdateRequestProcessor.finish()
+ (Drew Farris via koji)
+
+* SOLR-1973: Empty fields in XML update messages confuse DataImportHandler. (koji)
+
+* SOLR-2221: Use StrUtils.parseBool() to get values of boolean options in DIH.
+ true/on/yes (for TRUE) and false/off/no (for FALSE) can be used for sub-options
+ (debug, verbose, synchronous, commit, clean, optimize) for full/delta-import commands. (koji)
+
* SOLR-2310: getTimeElapsedSince() returns incorrect hour value when the elapse is over 60 hours
(tom liu via koji)
@@ -80,6 +96,11 @@ Bug Fixes
* SOLR-2330: solrconfig.xml files in example-DIH are broken. (Matt Parker, koji)
+* SOLR-1191: resolve DataImportHandler deltaQuery column against pk when pk
+ has a prefix (e.g. pk="book.id" deltaQuery="select id from ..."). More
+ useful error reporting when no match found (previously failed with a
+ NullPointerException in log and no clear user feedback). (gthb via yonik)
+
* SOLR-2116: Fix TikaConfig classloader bug in TikaEntityProcessor
(Martijn van Groningen via hossman)
Modified: lucene/dev/branches/realtime_search/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java (original)
+++ lucene/dev/branches/realtime_search/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandler.java Wed Mar 30 09:17:25 2011
@@ -27,6 +27,7 @@ import org.apache.solr.common.util.Conte
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.SystemIdResolver;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
@@ -41,8 +42,10 @@ import org.apache.solr.update.processor.
import org.apache.solr.util.plugin.SolrCoreAware;
import java.util.*;
+import java.io.StringReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.xml.sax.InputSource;
/**
* <p>
@@ -108,9 +111,9 @@ public class DataImportHandler extends R
String configLoc = (String) defaults.get("config");
if (configLoc != null && configLoc.length() != 0) {
processConfiguration(defaults);
-
- importer = new DataImporter(SolrWriter.getResourceAsString(core
- .getResourceLoader().openResource(configLoc)), core,
+ final InputSource is = new InputSource(core.getResourceLoader().openConfig(configLoc));
+ is.setSystemId(SystemIdResolver.createSystemIdFromResourceName(configLoc));
+ importer = new DataImporter(is, core,
dataSources, coreScopeSession);
}
}
@@ -162,7 +165,7 @@ public class DataImportHandler extends R
if (requestParams.dataConfig != null) {
try {
processConfiguration((NamedList) initArgs.get("defaults"));
- importer = new DataImporter(requestParams.dataConfig, req.getCore()
+ importer = new DataImporter(new InputSource(new StringReader(requestParams.dataConfig)), req.getCore()
, dataSources, coreScopeSession);
} catch (RuntimeException e) {
rsp.add("exception", DebugLogger.getStacktraceString(e));
Modified: lucene/dev/branches/realtime_search/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java (original)
+++ lucene/dev/branches/realtime_search/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java Wed Mar 30 09:17:25 2011
@@ -24,6 +24,8 @@ import org.apache.solr.schema.IndexSchem
import org.apache.solr.schema.SchemaField;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.SystemIdResolver;
+import org.apache.solr.common.util.XMLErrorLogger;
import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
@@ -33,6 +35,7 @@ import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
+import org.apache.commons.io.IOUtils;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
@@ -58,6 +61,7 @@ public class DataImporter {
}
private static final Logger LOG = LoggerFactory.getLogger(DataImporter.class);
+ private static final XMLErrorLogger XMLLOG = new XMLErrorLogger(LOG);
private Status status = Status.IDLE;
@@ -88,7 +92,7 @@ public class DataImporter {
coreScopeSession = new ConcurrentHashMap<String, Object>();
}
- DataImporter(String dataConfig, SolrCore core, Map<String, Properties> ds, Map<String, Object> session) {
+ DataImporter(InputSource dataConfig, SolrCore core, Map<String, Properties> ds, Map<String, Object> session) {
if (dataConfig == null)
throw new DataImportHandlerException(SEVERE,
"Configuration not found");
@@ -141,7 +145,7 @@ public class DataImporter {
* Used by tests
*/
void loadAndInit(String configStr) {
- loadDataConfig(configStr);
+ loadDataConfig(new InputSource(new StringReader(configStr)));
Map<String, DataConfig.Field> fields = new HashMap<String, DataConfig.Field>();
for (DataConfig.Entity entity : config.document.entities) {
initEntity(entity, fields, false);
@@ -168,19 +172,32 @@ public class DataImporter {
}
- private void loadDataConfig(String configFile) {
+ private void loadDataConfig(InputSource configFile) {
try {
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
- try {
- dbf.setXIncludeAware(true);
- dbf.setNamespaceAware(true);
- } catch( UnsupportedOperationException e ) {
- LOG.warn( "XML parser doesn't support XInclude option" );
+
+ // only enable xinclude, if a a SolrCore and SystemId is present (makes no sense otherwise)
+ if (core != null && configFile.getSystemId() != null) {
+ try {
+ dbf.setXIncludeAware(true);
+ dbf.setNamespaceAware(true);
+ } catch( UnsupportedOperationException e ) {
+ LOG.warn( "XML parser doesn't support XInclude option" );
+ }
}
+
DocumentBuilder builder = dbf.newDocumentBuilder();
- Document document = builder.parse(new InputSource(new StringReader(
- configFile)));
+ if (core != null)
+ builder.setEntityResolver(new SystemIdResolver(core.getResourceLoader()));
+ builder.setErrorHandler(XMLLOG);
+ Document document;
+ try {
+ document = builder.parse(configFile);
+ } finally {
+ // some XML parsers are broken and don't close the byte stream (but they should according to spec)
+ IOUtils.closeQuietly(configFile.getByteStream());
+ }
config = new DataConfig();
NodeList elems = document.getElementsByTagName("dataConfig");
Modified: lucene/dev/branches/realtime_search/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java (original)
+++ lucene/dev/branches/realtime_search/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java Wed Mar 30 09:17:25 2011
@@ -18,8 +18,13 @@ package org.apache.solr.handler.dataimpo
import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.common.ResourceLoader;
+import org.apache.solr.common.util.SystemIdResolver;
+import org.apache.solr.common.util.XMLErrorLogger;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.commons.io.IOUtils;
import javax.xml.transform.Source;
import javax.xml.transform.TransformerException;
@@ -50,6 +55,7 @@ import java.util.concurrent.atomic.Atomi
*/
public class XPathEntityProcessor extends EntityProcessorBase {
private static final Logger LOG = LoggerFactory.getLogger(XPathEntityProcessor.class);
+ private static final XMLErrorLogger xmllog = new XMLErrorLogger(LOG);
private static final Map<String, Object> END_MARKER = new HashMap<String, Object>();
@@ -107,12 +113,27 @@ public class XPathEntityProcessor extend
if (xslt != null) {
xslt = context.replaceTokens(xslt);
try {
- Source xsltSource = new StreamSource(xslt);
// create an instance of TransformerFactory
TransformerFactory transFact = TransformerFactory.newInstance();
- xslTransformer = transFact.newTransformer(xsltSource);
- LOG
- .info("Using xslTransformer: "
+ final SolrCore core = context.getSolrCore();
+ final StreamSource xsltSource;
+ if (core != null) {
+ final ResourceLoader loader = core.getResourceLoader();
+ transFact.setURIResolver(new SystemIdResolver(loader).asURIResolver());
+ xsltSource = new StreamSource(loader.openResource(xslt),
+ SystemIdResolver.createSystemIdFromResourceName(xslt));
+ } else {
+ // fallback for tests
+ xsltSource = new StreamSource(xslt);
+ }
+ transFact.setErrorListener(xmllog);
+ try {
+ xslTransformer = transFact.newTransformer(xsltSource);
+ } finally {
+ // some XML parsers are broken and don't close the byte stream (but they should according to spec)
+ IOUtils.closeQuietly(xsltSource.getInputStream());
+ }
+ LOG.info("Using xslTransformer: "
+ xslTransformer.getClass().getName());
} catch (Exception e) {
throw new DataImportHandlerException(SEVERE,
Modified: lucene/dev/branches/realtime_search/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathRecordReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathRecordReader.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathRecordReader.java (original)
+++ lucene/dev/branches/realtime_search/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/XPathRecordReader.java Wed Mar 30 09:17:25 2011
@@ -16,6 +16,7 @@
*/
package org.apache.solr.handler.dataimport;
+import org.apache.solr.common.util.XMLErrorLogger;
import javax.xml.stream.XMLInputFactory;
import static javax.xml.stream.XMLStreamConstants.*;
import javax.xml.stream.XMLStreamException;
@@ -25,6 +26,8 @@ import java.io.Reader;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* <p>
@@ -54,6 +57,9 @@ import java.util.regex.Pattern;
* @since solr 1.3
*/
public class XPathRecordReader {
+ private static final Logger LOG = LoggerFactory.getLogger(XPathRecordReader.class);
+ private static final XMLErrorLogger XMLLOG = new XMLErrorLogger(LOG);
+
private Node rootNode = new Node("/", null);
/**
@@ -622,6 +628,7 @@ public class XPathRecordReader {
static{
factory.setProperty(XMLInputFactory.IS_VALIDATING , Boolean.FALSE);
factory.setProperty(XMLInputFactory.SUPPORT_DTD , Boolean.FALSE);
+ factory.setXMLReporter(XMLLOG);
}
/**Implement this interface to stream records as and when one is found.
Modified: lucene/dev/branches/realtime_search/solr/contrib/extraction/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/contrib/extraction/CHANGES.txt?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/contrib/extraction/CHANGES.txt (original)
+++ lucene/dev/branches/realtime_search/solr/contrib/extraction/CHANGES.txt Wed Mar 30 09:17:25 2011
@@ -28,6 +28,10 @@ $Id:$
(No Changes)
+================== Release 3.2-dev ==================
+
+(No Changes)
+
================== Release 3.1-dev ==================
* SOLR-1902: Upgraded to Tika 0.8 and changed deprecated parse call
Modified: lucene/dev/branches/realtime_search/solr/contrib/uima/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/contrib/uima/CHANGES.txt?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/contrib/uima/CHANGES.txt (original)
+++ lucene/dev/branches/realtime_search/solr/contrib/uima/CHANGES.txt Wed Mar 30 09:17:25 2011
@@ -11,7 +11,21 @@ Such fields could be language, concepts,
UIMA Dependency
---------------
-uima-core, OpenCalaisAnnotator, WhitespaceTokenizer, HMMTagger, AlchemyAPIAnnotator
-Current Version: 2.3.1-SNAPSHOT rev. 999276
+uimaj-core v2.3.1
+OpenCalaisAnnotator v2.3.1-SNAPSHOT rev. 1062868
+HMMTagger v2.3.1-SNAPSHOT rev. 1062868
+AlchemyAPIAnnotator v2.3.1-SNAPSHOT rev. 1062868
+WhitespaceTokenizer v2.3.1-SNAPSHOT rev. 1076132
$Id$
+
+================== 3.2.0-dev ==================
+
+Test Cases:
+
+ * SOLR-2387: add mock annotators for improved testing,
+ (Tommaso Teofili via rmuir)
+
+================== 3.1.0-dev ==================
+
+Initial Release
Modified: lucene/dev/branches/realtime_search/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/SolrUIMAConfiguration.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/SolrUIMAConfiguration.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/SolrUIMAConfiguration.java (original)
+++ lucene/dev/branches/realtime_search/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/SolrUIMAConfiguration.java Wed Mar 30 09:17:25 2011
@@ -34,11 +34,11 @@ public class SolrUIMAConfiguration {
private String aePath;
- private Map<String, String> runtimeParameters;
+ private Map<String, Object> runtimeParameters;
public SolrUIMAConfiguration(String aePath, String[] fieldsToAnalyze, boolean fieldsMerging,
Map<String, Map<String, String>> typesFeaturesFieldsMapping,
- Map<String, String> runtimeParameters) {
+ Map<String, Object> runtimeParameters) {
this.aePath = aePath;
this.fieldsToAnalyze = fieldsToAnalyze;
this.fieldsMerging = fieldsMerging;
@@ -62,7 +62,7 @@ public class SolrUIMAConfiguration {
return aePath;
}
- public Map<String, String> getRuntimeParameters() {
+ public Map<String, Object> getRuntimeParameters() {
return runtimeParameters;
}
Modified: lucene/dev/branches/realtime_search/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/SolrUIMAConfigurationReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/SolrUIMAConfigurationReader.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/SolrUIMAConfigurationReader.java (original)
+++ lucene/dev/branches/realtime_search/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/SolrUIMAConfigurationReader.java Wed Mar 30 09:17:25 2011
@@ -105,15 +105,15 @@ public class SolrUIMAConfigurationReader
return map;
}
- private Map<String, String> readAEOverridingParameters() {
- Map<String, String> runtimeParameters = new HashMap<String, String>();
+ private Map<String, Object> readAEOverridingParameters() {
+ Map<String, Object> runtimeParameters = new HashMap<String, Object>();
Node uimaConfigNode = solrConfig.getNode(AE_RUNTIME_PARAMETERS_NODE_PATH, true);
if (uimaConfigNode.hasChildNodes()) {
NodeList overridingNodes = uimaConfigNode.getChildNodes();
for (int i = 0; i < overridingNodes.getLength(); i++) {
Node overridingNode = overridingNodes.item(i);
- if (overridingNode.getNodeType() != Node.TEXT_NODE) {
+ if (overridingNode.getNodeType() != Node.TEXT_NODE && overridingNode.getNodeType() != Node.COMMENT_NODE) {
runtimeParameters.put(overridingNode.getNodeName(), overridingNode.getTextContent());
}
}
Modified: lucene/dev/branches/realtime_search/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/ae/AEProviderFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/ae/AEProviderFactory.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/ae/AEProviderFactory.java (original)
+++ lucene/dev/branches/realtime_search/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/ae/AEProviderFactory.java Wed Mar 30 09:17:25 2011
@@ -43,7 +43,7 @@ public class AEProviderFactory {
}
public synchronized AEProvider getAEProvider(String core, String aePath,
- Map<String, String> runtimeParameters) {
+ Map<String, Object> runtimeParameters) {
String key = new StringBuilder(core).append(aePath).toString();
if (providerCache.get(key) == null) {
providerCache.put(key, new OverridingParamsAEProvider(aePath, runtimeParameters));
Modified: lucene/dev/branches/realtime_search/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/ae/OverridingParamsAEProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/ae/OverridingParamsAEProvider.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/ae/OverridingParamsAEProvider.java (original)
+++ lucene/dev/branches/realtime_search/solr/contrib/uima/src/main/java/org/apache/solr/uima/processor/ae/OverridingParamsAEProvider.java Wed Mar 30 09:17:25 2011
@@ -43,9 +43,9 @@ public class OverridingParamsAEProvider
private AnalysisEngine cachedAE;
- private Map<String, String> runtimeParameters;
+ private Map<String, Object> runtimeParameters;
- public OverridingParamsAEProvider(String aeFilePath, Map<String, String> runtimeParameters) {
+ public OverridingParamsAEProvider(String aeFilePath, Map<String, Object> runtimeParameters) {
this.aeFilePath = aeFilePath;
this.runtimeParameters = runtimeParameters;
}
@@ -63,9 +63,11 @@ public class OverridingParamsAEProvider
/* iterate over each AE (to set runtime parameters) */
for (String attributeName : runtimeParameters.keySet()) {
+ Object val = getRuntimeValue(desc, attributeName);
desc.getAnalysisEngineMetaData().getConfigurationParameterSettings().setParameterValue(
- attributeName, runtimeParameters.get(attributeName));
- log.info(new StringBuilder("setting ").append(attributeName).append(" : ").append(
+ attributeName, val);
+ if (log.isDebugEnabled())
+ log.debug(new StringBuilder("setting ").append(attributeName).append(" : ").append(
runtimeParameters.get(attributeName)).toString());
}
// create AE here
@@ -86,4 +88,30 @@ public class OverridingParamsAEProvider
return cachedAE;
}
+ /* create the value to inject in the runtime parameter depending on its declared type */
+ private Object getRuntimeValue(AnalysisEngineDescription desc, String attributeName)
+ throws ClassNotFoundException {
+ String type = desc.getAnalysisEngineMetaData().getConfigurationParameterDeclarations().
+ getConfigurationParameter(null, attributeName).getType();
+ // TODO : do it via reflection ? i.e. Class paramType = Class.forName(type)...
+ Object val = null;
+ Object runtimeValue = runtimeParameters.get(attributeName);
+ if (runtimeValue!=null) {
+ if ("String".equals(type)) {
+ val = String.valueOf(runtimeValue);
+ }
+ else if ("Integer".equals(type)) {
+ val = Integer.valueOf(runtimeValue.toString());
+ }
+ else if ("Boolean".equals(type)) {
+ val = Boolean.valueOf(runtimeValue.toString());
+ }
+ else if ("Float".equals(type)) {
+ val = Float.valueOf(runtimeValue.toString());
+ }
+ }
+
+ return val;
+ }
+
}
\ No newline at end of file
Modified: lucene/dev/branches/realtime_search/solr/contrib/uima/src/test/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/contrib/uima/src/test/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/contrib/uima/src/test/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java (original)
+++ lucene/dev/branches/realtime_search/solr/contrib/uima/src/test/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessorTest.java Wed Mar 30 09:17:25 2011
@@ -70,8 +70,6 @@ public class UIMAUpdateRequestProcessorT
@Test
public void testProcessing() throws Exception {
- // this test requires an internet connection (e.g. opencalais api)
- checkInternetConnection();
addDoc(adoc(
"id",
@@ -83,31 +81,29 @@ public class UIMAUpdateRequestProcessorT
+ " attached if you need it, but it is also committed to trunk and 3_x branch."
+ " Last Lucene European Conference has been held in Prague."));
assertU(commit());
- assertQ(req("language:english"), "//*[@numFound='1']");
+ assertQ(req("sentence:*"), "//*[@numFound='1']");
+ assertQ(req("sentiment:*"), "//*[@numFound='0']");
+ assertQ(req("entity:Prague"), "//*[@numFound='1']");
}
@Test
- public void testTwoUpdates() {
- // this test requires an internet connection (e.g. opencalais api)
- checkInternetConnection();
-
- try {
- addDoc(adoc("id", "1", "text", "The Apache Software Foundation is happy to announce "
- + "BarCampApache Sydney, Australia, the first ASF-backed event in the Southern "
- + "Hemisphere!"));
- assertU(commit());
- assertQ(req("language:english"), "//*[@numFound='1']");
-
- addDoc(adoc("id", "2", "text", "Taking place 11th December 2010 at the University "
- + "of Sydney's Darlington Centre, the BarCampApache \"unconference\" will be"
- + " attendee-driven, facilitated by members of the Apache community and will "
- + "focus on the Apache..."));
- assertU(commit());
- assertQ(req("language:english"), "//*[@numFound='2']");
-
- } catch (Exception e) {
- assumeNoException("Multiple updates on same instance didn't work", e);
- }
+ public void testTwoUpdates() throws Exception {
+
+ addDoc(adoc("id", "1", "text", "The Apache Software Foundation is happy to announce "
+ + "BarCampApache Sydney, Australia, the first ASF-backed event in the Southern "
+ + "Hemisphere!"));
+ assertU(commit());
+ assertQ(req("sentence:*"), "//*[@numFound='1']");
+
+ addDoc(adoc("id", "2", "text", "Taking place 11th December 2010 at the University "
+ + "of Sydney's Darlington Centre, the BarCampApache \"unconference\" will be"
+ + " attendee-driven, facilitated by members of the Apache community and will "
+ + "focus on the Apache..."));
+ assertU(commit());
+ assertQ(req("sentence:*"), "//*[@numFound='2']");
+
+ assertQ(req("sentiment:positive"), "//*[@numFound='1']");
+ assertQ(req("entity:Apache"), "//*[@numFound='2']");
}
private void addDoc(String doc) throws Exception {
@@ -125,14 +121,4 @@ public class UIMAUpdateRequestProcessorT
handler.handleRequestBody(req, new SolrQueryResponse());
}
- private void checkInternetConnection() {
- try {
- URLConnection conn = new URL("http://www.apache.org/").openConnection();
- conn.setConnectTimeout(5000);
- conn.setReadTimeout(5000);
- conn.connect();
- } catch (Exception ex) {
- assumeNoException("This test requires an internet connection", ex);
- }
- }
}
Modified: lucene/dev/branches/realtime_search/solr/contrib/uima/src/test/resources/solr-uima/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/contrib/uima/src/test/resources/solr-uima/conf/schema.xml?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/contrib/uima/src/test/resources/solr-uima/conf/schema.xml (original)
+++ lucene/dev/branches/realtime_search/solr/contrib/uima/src/test/resources/solr-uima/conf/schema.xml Wed Mar 30 09:17:25 2011
@@ -562,11 +562,9 @@
-->
<field name="language" type="string" indexed="true" stored="true" required="false"/>
- <field name="concept" type="string" indexed="true" stored="true" multiValued="true" required="false"/>
- <field name="keyword" type="string" indexed="true" stored="true" multiValued="true" required="false"/>
- <field name="suggested_category" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
<field name="sentence" type="text" indexed="true" stored="true" multiValued="true" required="false" />
- <dynamicField name="entity*" type="text" indexed="true" stored="true" multiValued="true"/>
+ <field name="sentiment" type="string" indexed="true" stored="true" multiValued="true"/>
+ <field name="entity" type="text" indexed="true" stored="true" multiValued="true"/>
<!--
Dynamic field definitions. If a field name is not found,
Modified: lucene/dev/branches/realtime_search/solr/contrib/uima/src/test/resources/solr-uima/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/contrib/uima/src/test/resources/solr-uima/conf/solrconfig.xml?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/contrib/uima/src/test/resources/solr-uima/conf/solrconfig.xml (original)
+++ lucene/dev/branches/realtime_search/solr/contrib/uima/src/test/resources/solr-uima/conf/solrconfig.xml Wed Mar 30 09:17:25 2011
@@ -1064,38 +1064,21 @@
<uimaConfig>
<runtimeParameters>
- <keyword_apikey>04490000a72fe7ec5cb3497f14e77f338c86f2fe</keyword_apikey>
- <concept_apikey>04490000a72fe7ec5cb3497f14e77f338c86f2fe</concept_apikey>
- <lang_apikey>04490000a72fe7ec5cb3497f14e77f338c86f2fe</lang_apikey>
- <cat_apikey>04490000a72fe7ec5cb3497f14e77f338c86f2fe</cat_apikey>
- <entities_apikey>04490000a72fe7ec5cb3497f14e77f338c86f2fe</entities_apikey>
- <oc_licenseID>g6h9zamsdtwhb93nc247ecrs</oc_licenseID>
+ <ngramsize>3</ngramsize>
</runtimeParameters>
- <analysisEngine>/org/apache/uima/desc/OverridingParamsExtServicesAE.xml</analysisEngine>
+ <analysisEngine>/TestAE.xml</analysisEngine>
<analyzeFields merge="false">text</analyzeFields>
<fieldMapping>
- <type name="org.apache.uima.alchemy.ts.categorization.Category">
- <map feature="text" field="suggested_category"/>
- </type>
- <type name="org.apache.uima.alchemy.ts.concept.ConceptFS">
- <map feature="text" field="concept"/>
+ <type name="org.apache.uima.SentenceAnnotation">
+ <map feature="coveredText" field="sentence"/>
</type>
- <type name="org.apache.uima.alchemy.ts.entity.BaseEntity">
- <map feature="text" field="entity"/>
+ <type name="org.apache.solr.uima.ts.SentimentAnnotation">
+ <map feature="mood" field="sentiment"/>
</type>
- <type name="org.apache.uima.calais.BaseType">
+ <type name="org.apache.solr.uima.ts.EntityAnnotation">
<map feature="coveredText" field="entity"/>
</type>
- <type name="org.apache.uima.alchemy.ts.keywords.KeywordFS">
- <map feature="text" field="keyword"/>
- </type>
- <type name="org.apache.uima.alchemy.ts.language.LanguageFS">
- <map feature="language" field="language"/>
- </type>
- <type name="org.apache.uima.SentenceAnnotation">
- <map feature="coveredText" field="sentence"/>
- </type>
</fieldMapping>
</uimaConfig>
-
+
</config>
Modified: lucene/dev/branches/realtime_search/solr/example/README.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/example/README.txt?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/example/README.txt (original)
+++ lucene/dev/branches/realtime_search/solr/example/README.txt Wed Mar 30 09:17:25 2011
@@ -35,13 +35,15 @@ See also README.txt in the solr subdirec
http://wiki.apache.org/solr/SolrResources for a list of tutorials and
introductory articles.
-NOTE: This Solr example server references SolrCell jars outside of the server
-directory with <lib> statements in the solrconfig.xml. If you make a copy of
-this example server and wish to use the ExtractingRequestHandler (SolrCell),
+NOTE: This Solr example server references certain Solr jars outside of
+this server directory for non-core modules with <lib> statements in
+solrconfig.xml. If you make a copy of this example server and wish
+to use the ExtractingRequestHandler (SolrCell), DataImportHandler (DIH),
+UIMA, the clustering component, or other modules in "contrib",
you will need to copy the required jars into solr/lib or update the paths to
the jars in your solrconfig.xml.
-By default, start.jar starts Solr in Jetty using the default SolrHome
+By default, start.jar starts Solr in Jetty using the default solr home
directory of "./solr/" -- To run other example configurations, you can
speciy the solr.solr.home system property when starting jetty...
Modified: lucene/dev/branches/realtime_search/solr/example/exampledocs/test_utf8.sh
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/example/exampledocs/test_utf8.sh?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/example/exampledocs/test_utf8.sh (original)
+++ lucene/dev/branches/realtime_search/solr/example/exampledocs/test_utf8.sh Wed Mar 30 09:17:25 2011
@@ -53,12 +53,13 @@ else
fi
-#A codepoint outside of the BMP
-CODEPOINT='\\U00100058'
+#A unicode character outside of the BMP (a circle with an x inside)
+CHAR="ð"
+CODEPOINT='0x10308'
#URL encoded UTF8 of the codepoint
-URL_UTF8='%F4%80%81%98'
+URL_UTF8='%F0%90%8C%88'
#expected return of the python writer (currently uses UTF-16 surrogates)
-EXPECTED='\\udbc0\\udc58'
+EXPECTED='\\ud800\\udf08'
curl "$URL/select?q=$URL_UTF8&echoParams=explicit&wt=python" 2> /dev/null | grep $EXPECTED > /dev/null 2>&1
if [ $? = 0 ]; then
@@ -81,3 +82,12 @@ else
echo "ERROR: HTTP POST + URL params is not accepting UTF-8 beyond the basic multilingual plane"
fi
+#curl "$URL/select?q=$URL_UTF8&echoParams=explicit&wt=json" 2> /dev/null | od -tx1 -w1000 | sed 's/ //g' | grep 'f4808198' > /dev/null 2>&1
+curl "$URL/select?q=$URL_UTF8&echoParams=explicit&wt=json" 2> /dev/null | grep "$CHAR" > /dev/null 2>&1
+if [ $? = 0 ]; then
+ echo "Response correctly returns UTF-8 beyond the basic multilingual plane"
+else
+ echo "ERROR: Response can't return UTF-8 beyond the basic multilingual plane"
+fi
+
+
Modified: lucene/dev/branches/realtime_search/solr/example/exampledocs/utf8-example.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/example/exampledocs/utf8-example.xml?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/example/exampledocs/utf8-example.xml (original)
+++ lucene/dev/branches/realtime_search/solr/example/exampledocs/utf8-example.xml Wed Mar 30 09:17:25 2011
@@ -34,8 +34,8 @@
<field name="features">eaiou with umlauts: ëäïöü</field>
<field name="features">tag with escaped chars: <nicetag/></field>
<field name="features">escaped ampersand: Bonnie & Clyde</field>
+ <field name="features">Outside the BMP:ð codepoint=10308, a circle with an x inside. UTF8=f0908c88 UTF16=d800 df08</field>
<field name="price">0</field>
- <!-- no popularity, get the default from schema.xml -->
<field name="inStock">true</field>
</doc>
</add>
Modified: lucene/dev/branches/realtime_search/solr/example/solr/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/example/solr/conf/schema.xml?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/example/solr/conf/schema.xml (original)
+++ lucene/dev/branches/realtime_search/solr/example/solr/conf/schema.xml Wed Mar 30 09:17:25 2011
@@ -505,6 +505,7 @@
<dynamicField name="*_s" type="string" indexed="true" stored="true"/>
<dynamicField name="*_l" type="long" indexed="true" stored="true"/>
<dynamicField name="*_t" type="text" indexed="true" stored="true"/>
+ <dynamicField name="*_txt" type="text" indexed="true" stored="true" multiValued="true"/>
<dynamicField name="*_b" type="boolean" indexed="true" stored="true"/>
<dynamicField name="*_f" type="float" indexed="true" stored="true"/>
<dynamicField name="*_d" type="double" indexed="true" stored="true"/>