You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2013/01/18 19:31:23 UTC
svn commit: r1435287 [25/41] - in /lucene/dev/branches/LUCENE-2878: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/libraries/
dev-tools/idea/lucene/analysis/icu/ dev-tools/maven/
dev-tools/maven/lucene/benchmark/ dev-tools/maven/solr/ dev-tools/...
Modified: lucene/dev/branches/LUCENE-2878/solr/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/build.xml?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/build.xml (original)
+++ lucene/dev/branches/LUCENE-2878/solr/build.xml Fri Jan 18 18:30:54 2013
@@ -26,7 +26,7 @@
<echo message="Use 'ant clean' to clean compiled files." />
<echo message="Use 'ant compile' to compile the source code." />
<echo message="Use 'ant dist' to build the project WAR and JAR files." />
- <echo message="Use 'ant javadocs' to build javadocs under build/docs/api" />
+ <echo message="Use 'ant documentation' to build documentation." />
<echo message="Use 'ant generate-maven-artifacts' to generate maven artifacts." />
<echo message="Use 'ant package' to generate zip, tgz for distribution." />
<!--<echo message="Use 'ant luke' to start luke. see: http://luke.googlecode.com" />-->
@@ -131,14 +131,48 @@
depends="compile-core, compile-contrib"/>
<target name="test" description="Validate, then run core, solrj, and contrib unit tests."
depends="test-core, test-contrib"/>
+ <!-- "-clover.load" is *not* a useless dependency. do not remove -->
<target name="test-core" description="Runs the core and solrj unit tests."
- depends="test-solr-core, test-solrj"/>
+ depends="-clover.load, test-solr-core, test-solrj"/>
+ <target name="pitest" description="Validate, then run core, solrj, and contrib unit tests."
+ depends="pitest-core, pitest-contrib"/>
<target name="compile-test" description="Compile unit tests."
depends="compile-solr-test-framework, compile-test-solr-core, compile-test-solrj, compile-test-contrib"/>
<target name="javadocs" description="Calls javadocs-all, javadocs-solrj, and javadocs-test-framework"
- depends="javadocs-all,javadocs-solrj,javadocs-test-framework"/>
+ depends="define-lucene-javadoc-url,javadocs-solr-core,javadocs-solrj,javadocs-test-framework,javadocs-contrib"/>
+ <target name="documentation" description="Generate all documentation"
+ depends="javadocs,changes-to-html,process-webpages"/>
<target name="compile-core" depends="compile-solr-core" unless="solr.core.compiled"/>
+ <target name="process-webpages" depends="define-lucene-javadoc-url,resolve-groovy,resolve-pegdown">
+ <makeurl property="process-webpages.buildfiles" separator="|">
+ <fileset dir="." includes="core/build.xml,test-framework/build.xml,solrj/build.xml,contrib/**/build.xml"/>
+ </makeurl>
+ <!--
+ The XSL input file is ignored completely, but XSL expects one to be given,
+ so we pass ourself (${ant.file}) here. The list of module build.xmls is given
+ via string parameter, that must be splitted by the XSL at '|'.
+ -->
+ <xslt in="${ant.file}" out="${javadoc.dir}/index.html" style="site/xsl/index.xsl" force="true">
+ <outputproperty name="method" value="html"/>
+ <outputproperty name="version" value="4.0"/>
+ <outputproperty name="encoding" value="UTF-8"/>
+ <outputproperty name="indent" value="yes"/>
+ <param name="buildfiles" expression="${process-webpages.buildfiles}"/>
+ <param name="version" expression="${version}"/>
+ <param name="luceneJavadocUrl" expression="${lucene.javadoc.url}"/>
+ </xslt>
+
+ <pegdown todir="${javadoc.dir}">
+ <fileset dir="." includes="SYSTEM_REQUIREMENTS.txt"/>
+ <globmapper from="*.txt" to="*.html"/>
+ </pegdown>
+
+ <copy todir="${javadoc.dir}">
+ <fileset dir="site/html" includes="**/*"/>
+ </copy>
+ </target>
+
<!-- Solr core targets -->
<target name="test-solr-core" description="Test solr core">
<ant dir="core" target="test" inheritAll="false">
@@ -152,16 +186,28 @@
<propertyset refid="uptodate.and.compiled.properties"/>
</ant>
</target>
- <target name="javadocs-solrj">
- <ant dir="solrj" target="javadocs" inheritAll="false">
- <propertyset refid="uptodate.and.compiled.properties"/>
- </ant>
- </target>
<!-- Solr contrib targets -->
<target name="test-contrib" description="Run contrib unit tests.">
<contrib-crawl target="test" failonerror="true"/>
</target>
+
+ <!-- Pitest targets -->
+ <target name="pitest-core" description="PiTest solr core">
+ <ant dir="core" target="pitest" inheritAll="false">
+ <propertyset refid="uptodate.and.compiled.properties"/>
+ </ant>
+ </target>
+
+ <target name="pitest-solrj" description="PiTest java client">
+ <ant dir="solrj" target="pitest" inheritAll="false">
+ <propertyset refid="uptodate.and.compiled.properties"/>
+ </ant>
+ </target>
+
+ <target name="pitest-contrib" description="Run contrib PiTests.">
+ <contrib-crawl target="pitest" failonerror="false"/>
+ </target>
<!-- test-framework targets -->
<target name="javadocs-test-framework">
@@ -180,10 +226,10 @@
<exclude name="example/start.jar" />
<exclude name="example/exampledocs/post.jar" />
<exclude name="example/solr-webapp/**" />
+ <exclude name="package/**"/>
</additional-excludes>
<additional-filters>
<replaceregex pattern="jetty([^/]+)$" replace="jetty" flags="gi" />
- <replaceregex pattern="apache-solr-commons-csv-([^/]+)$" replace="apache-solr-commons-csv" flags="gi" />
<replaceregex pattern="slf4j-([^/]+)$" replace="slf4j" flags="gi" />
<replaceregex pattern="(bcmail|bcprov)-([^/]+)$" replace="\1" flags="gi" />
</additional-filters>
@@ -201,9 +247,12 @@
<include name="jdk-deprecated.txt" />
<include name="commons-io.txt" />
<include name="executors.txt" />
+ <include name="servlet-api.txt" />
</apiFileSet>
<fileset dir="${basedir}/build">
<include name="**/*.class" />
+ <!-- violates the servlet-api restrictions, but it is safe to do so in this test: -->
+ <exclude name="solr-solrj/classes/test/org/apache/solr/client/solrj/impl/BasicHttpSolrServerTest$DebugServlet.class"/>
</fileset>
</forbidden-apis>
</target>
@@ -223,13 +272,15 @@
</target>
<!-- rat sources -->
- <target name="rat-sources">
- <sequential>
- <ant dir="core" target="rat-sources" inheritall="false"/>
- <ant dir="solrj" target="rat-sources" inheritall="false"/>
- <ant dir="test-framework" target="rat-sources" inheritall="false"/>
- <contrib-crawl target="rat-sources" failonerror="true"/>
- </sequential>
+ <!-- rat-sources-typedef is *not* a useless dependency. do not remove -->
+ <target name="rat-sources" depends="rat-sources-typedef">
+ <subant target="rat-sources" inheritall="false" >
+ <propertyset refid="uptodate.and.compiled.properties"/>
+ <fileset dir="core" includes="build.xml"/>
+ <fileset dir="solrj" includes="build.xml"/>
+ <fileset dir="test-framework" includes="build.xml"/>
+ </subant>
+ <contrib-crawl target="rat-sources" failonerror="true"/>
</target>
<!-- Clean targets -->
@@ -267,7 +318,18 @@
<target name="dist"
description="Creates the Solr distribution files."
- depends="dist-solrj, dist-core, dist-test-framework, dist-contrib, dist-war" />
+ depends="-dist-common, dist-war" />
+
+ <target name="dist-excl-slf4j"
+ description="Creates the Solr distribution files without slf4j API or bindings."
+ depends="-dist-common, dist-war-excl-slf4j" />
+
+ <target name="-dist-common"
+ depends="dist-solrj, dist-core, dist-test-framework, dist-contrib">
+ <!-- Targets common to dist and dist-excl-slf4j.
+ Only usable as a dependency
+ -->
+ </target>
<target name="dist-test-framework" depends="init-dist"
description="Creates the Solr test-framework JAR.">
@@ -290,10 +352,10 @@
</target>
<target name="dist-war-excl-slf4j"
- description="Creates a Solr WAR Distribution file, excluding slf4j bindings.">
+ description="Creates a Solr WAR Distribution file, excluding slf4j API and bindings.">
<ant dir="webapp" target="dist" inheritall="false">
<propertyset refid="uptodate.and.compiled.properties"/>
- <property name="exclude.from.war" value="*over-slf4j*,slf4j-jdk14*,log4j-*" />
+ <property name="exclude.from.war" value="*slf4j*,log4j-*" />
<property name="solr.war.suffix" value="-excl-slf4j" />
</ant>
</target>
@@ -302,7 +364,20 @@
<target name="prepare-release" depends="prepare-release-no-sign, sign-artifacts"/>
<!-- make a distribution -->
- <target name="package" depends="package-src-tgz,create-package"/>
+ <target name="package" depends="package-src-tgz,create-package,-dist-changes,-dist-keys"/>
+
+ <!-- copy changes/ to the release folder -->
+ <target name="-dist-changes">
+ <copy todir="${package.dir}/changes">
+ <fileset dir="build/docs/changes"/>
+ </copy>
+ </target>
+
+ <!-- copy KEYS to the release folder -->
+ <target name="-dist-keys">
+ <get src="http://people.apache.org/keys/group/lucene.asc"
+ dest="${package.dir}/KEYS"/>
+ </target>
<!-- Makes a tarball from running "svn export" at the root level. -->
<!-- Copies NOTICE.txt and LICENSE.txt from solr/ to the root level. -->
@@ -318,6 +393,10 @@
<!-- Exclude clover license files incompatible with the ASL -->
<delete dir="${svn.export.dir}/lucene/tools/clover"/>
+ <build-changes changes.src.file="${svn.export.dir}/solr/CHANGES.txt"
+ changes.target.dir="${svn.export.dir}/solr/docs/changes"
+ changes.product="SOLR"/>
+
<tar destfile="${source.package.file}" compression="gzip" longfile="gnu">
<tarfileset dir="${svn.export.dir}"
prefix="${fullnamever}"
@@ -366,7 +445,7 @@
<target name="create-package"
description="Packages the Solr Binary Distribution"
- depends="init-dist, dist, example, javadocs">
+ depends="init-dist, dist, example, documentation">
<mkdir dir="${dest}/${fullnamever}"/>
<delete includeemptydirs="true">
<fileset dir="${example}/solr-webapp" includes="**/*"/>
@@ -383,8 +462,8 @@
<tar destfile="${package.dir}/${fullnamever}.tgz" compression="gzip" longfile="gnu">
<tarfileset dir="."
prefix="${fullnamever}"
- includes="LICENSE.txt NOTICE.txt CHANGES.txt README.txt example/**
- client/README.txt client/ruby/solr-ruby/**
+ includes="LICENSE.txt NOTICE.txt CHANGES.txt README.txt SYSTEM_REQUIREMENTS.txt
+ example/** client/README.txt client/ruby/solr-ruby/**
contrib/**/lib/** contrib/**/README.txt
licenses/**"
excludes="lib/README.committers.txt **/data/ **/logs/*
@@ -404,7 +483,7 @@
dist/solrj-lib/*
dist/test-framework/**"
excludes="**/*.tgz **/*.zip **/*.md5 **/*src*.jar **/*docs*.jar **/*.sha1" />
- <tarfileset dir="${dest}/docs"
+ <tarfileset dir="${javadoc.dir}"
prefix="${fullnamever}/docs" />
</tar>
<make-checksums file="${package.dir}/${fullnamever}.tgz"/>
@@ -442,15 +521,15 @@
</zip>
<make-checksums file="${package.dir}/${fullnamever}.zip"/>
</target>
+
+ <target name="changes-to-html">
+ <build-changes changes.product="SOLR"/>
+ </target>
<target name="sign-artifacts">
<sign-artifacts-macro artifacts.dir="${package.dir}"/>
</target>
- <target name="javadocs-dep">
- <!-- NOOP -->
- </target>
-
<target name="resolve" depends="resolve-example">
<sequential>
<ant dir="core" target="resolve" inheritall="false">
@@ -466,64 +545,23 @@
</sequential>
</target>
- <!-- since we build across all contribs, we must ensure all deps
- are resolved -->
- <target name="javadocs-all"
- depends="resolve,prep-lucene-jars,javadocs-dep,lucene-javadocs,define-lucene-javadoc-url"
- description="Generate javadoc for core, java client and contrib">
- <sequential>
- <mkdir dir="${dest}/docs/api"/>
-
- <!-- TODO: optimize this, thats stupid here: -->
- <subant target="module-jars-to-solr">
- <fileset dir="contrib/analysis-extras" includes="build.xml"/>
- </subant>
-
- <!-- TODO: optimize this, thats stupid here: -->
- <subant target="module-jars-to-solr">
- <fileset dir="contrib/uima" includes="build.xml"/>
- </subant>
-
- <path id="javadoc.classpath">
- <path refid="test.classpath"/>
- <fileset dir="${dest}/contrib">
- <include name="**/lucene-libs/**/*.jar"/>
- </fileset>
- <fileset dir="contrib">
- <include name="**/lib/**/*.jar"/>
- <exclude name="**/analysis-extras/lib/**/*icu4j*.jar"/> <!-- extraction/lib/ has this one -->
- </fileset>
- <pathelement location="${dest}/solr-solrj/classes/java"/>
- </path>
-
- <solr-invoke-javadoc destdir="${dest}/docs/api"
- overview="core/src/java/overview.html">
- <solrsources>
- <packageset dir="core/src/java" />
- <packageset dir="solrj/src/java" />
- <packageset dir="contrib/analysis-extras/src/java"/>
- <packageset dir="contrib/clustering/src/java"/>
- <packageset dir="contrib/dataimporthandler/src/java"/>
- <packageset dir="contrib/dataimporthandler-extras/src/java"/>
- <packageset dir="contrib/extraction/src/java"/>
- <packageset dir="contrib/langid/src/java"/>
- <packageset dir="contrib/uima/src/java"/>
- <group title="Core" packages="org.apache.*" />
- <group title="SolrJ" packages="org.apache.solr.common.*,org.apache.solr.client.solrj.*,org.apache.zookeeper.*" />
- <group title="contrib: Clustering" packages="org.apache.solr.handler.clustering*" />
- <group title="contrib: DataImportHandler" packages="org.apache.solr.handler.dataimport*" />
- <group title="contrib: Solr Cell" packages="org.apache.solr.handler.extraction*" />
- <group title="contrib: Solr LangId" packages="org.apache.solr.update.processor.LanguageIdentifier*,org.apache.solr.update.processor.LangIdParams*,org.apache.solr.update.processor.DetectedLanguage*" />
- <group title="contrib: Solr UIMA" packages="org.apache.solr.uima*" />
- </solrsources>
- </solr-invoke-javadoc>
- </sequential>
- </target>
-
+ <target name="documentation-lint" depends="-ecj-javadoc-lint,-documentation-lint,-documentation-lint-unsupported"
+ description="Validates the generated documentation (HTML errors, broken links,...)"/>
+
<!-- TODO: does solr have any other docs we should check? -->
<!-- TODO: also integrate checkJavaDocs.py, which does more checks -->
- <target name="javadocs-lint" depends="javadocs">
- <check-broken-links dir="build/docs"/>
+ <target name="-documentation-lint" if="documentation-lint.supported" depends="documentation">
+ <jtidy-macro>
+ <!-- NOTE: must currently exclude deprecated-list due to a javadocs bug (as of 1.7.0_09)
+ javadocs generates invalid XML if you deprecate a method that takes a parameter
+ with a generic type -->
+ <fileset dir="build/docs" includes="**/*.html" excludes="**/deprecated-list.html"/>
+ </jtidy-macro>
+ <echo message="Checking for broken links..."/>
+ <check-broken-links dir="${javadoc.dir}"/>
+ <echo message="Checking for malformed docs..."/>
+ <!-- TODO: add missing package.htmls and bump this to level=package -->
+ <check-missing-javadocs dir="${javadoc.dir}" level="none"/>
<!-- prevent the modules without problems from getting worse -->
<check-missing-javadocs dir="${javadoc.dir}/solr-analysis-extras" level="package"/>
<check-missing-javadocs dir="${javadoc.dir}/solr-cell" level="package"/>
@@ -535,7 +573,18 @@
<check-missing-javadocs dir="${javadoc.dir}/solr-test-framework" level="package"/>
</target>
- <target name="generate-maven-artifacts" depends="install-maven-tasks">
+ <target name="-ecj-javadoc-lint" depends="documentation,compile-solr-test-framework,-ecj-resolve">
+ <subant target="-ecj-javadoc-lint" failonerror="true" inheritall="false">
+ <propertyset refid="uptodate.and.compiled.properties"/>
+ <fileset dir="core" includes="build.xml"/>
+ <fileset dir="solrj" includes="build.xml"/>
+ <fileset dir="test-framework" includes="build.xml"/>
+ </subant>
+ <contrib-crawl target="-ecj-javadoc-lint"/>
+ </target>
+
+ <!-- install-maven-tasks and define-lucene-javadoc-url are *not* a useless dependencies. Do not remove! -->
+ <target name="generate-maven-artifacts" depends="install-maven-tasks,define-lucene-javadoc-url">
<sequential>
<subant target="dist-maven" inheritall="false" >
<propertyset refid="uptodate.and.compiled.properties"/>
@@ -547,110 +596,32 @@
<contrib-crawl target="dist-maven"/>
</sequential>
</target>
+
+ <!-- only used by validate below, it needs the lucene artifacts installed: can we do this better?! -->
+ <target name="-generate-lucene-maven-artifacts">
+ <subant target="generate-maven-artifacts" inheritall="false">
+ <propertyset refid="uptodate.and.compiled.properties"/>
+ <fileset dir="${common.dir}" includes="build.xml"/>
+ </subant>
+ </target>
-
+ <target name="-validate-maven-dependencies" depends="compile-tools, install-maven-tasks, load-custom-tasks">
+ <sequential>
+ <subant target="-validate-maven-dependencies" failonerror="true" inheritall="false">
+ <propertyset refid="uptodate.and.compiled.properties"/>
+ <fileset dir="core" includes="build.xml"/>
+ <fileset dir="solrj" includes="build.xml"/>
+ <fileset dir="test-framework" includes="build.xml"/>
+ <fileset dir="webapp" includes="build.xml"/>
+ </subant>
+ <contrib-crawl target="-validate-maven-dependencies"/>
+ </sequential>
+ </target>
+
<!-- ========================================================================= -->
<!-- ========================= COMMITTERS' HELPERS =========================== -->
<!-- ========================================================================= -->
-
- <target name="stub-factories" depends="dist-core,lucene-jars-to-solr"
- description="Generates stub factories as needed">
-
- <path id="stub.jars">
- <!-- this needs to be a list of all jars that might contain
- classes we want to build factories for
- -->
- <fileset dir="${lucene-libs}">
- <include name="*.jar"/>
- </fileset>
- <fileset dir="${dist}">
- <include name="*.jar"/>
- <exclude name="*solrj*.jar"/>
- </fileset>
- </path>
- <pathconvert property="jar.list" pathsep=" " refid="stub.jars" />
- <path id="missing.factories.classpath">
- <pathelement path="${common.dir}/contrib/queries/lib/jakarta-regexp-1.4.jar"/>
- <path refid="test.classpath"/>
- </path>
- <property name="stub.list" value="${dest}/need-stub-factories.txt" />
- <java fork="false"
- classname="org.apache.solr.util.SuggestMissingFactories"
- logError="true"
- failonerror="true"
- classpathref="missing.factories.classpath"
- output="${stub.list}">
- <arg line="${jar.list}" />
- </java>
- <fail unless="stub.src.path">...
-
- This task requires that the property 'stub.src.path' be set.
-
- It must contain a "path" listing directories containing source
- files that this task should use when looking for classes that
- need factories created, the format is platform specific --
- typically it is colon seperated in Unix, semi-colon seperated
- on windows, ie:
-
- ant stub-factories -Dstub.src.path="core/src:../lucene/contrib:../lucene/core/src/java:../lucene/analysis"
-
- FYI: The file ${stub.list} contains a list of classes
- that seem to need stub factories. (if java files can be found to
- use as guides for creating them).
- </fail>
-
- <pathconvert pathsep=" " property="stub.src.dirs">
- <path>
- <pathelement path="${stub.src.path}"/>
- </path>
- </pathconvert>
- <exec executable="perl"
- dir="core/src/java/org/apache/solr/analysis/"
- failonerror="true">
- <arg value="${common-solr.dir}/dev-tools/stub-analysis-factory-maker.pl"/>
- <redirector input="${stub.list}">
- <!-- place to put special case classes we want to ignore -->
- <inputfilterchain>
- <linecontainsregexp negate="true">
- <!-- only for internal Solr highlighting purposes -->
- <regexp pattern="TokenOrderingFilter"/>
- </linecontainsregexp>
- <linecontainsregexp negate="true">
- <!-- no way to leverage this in Solr -->
- <regexp pattern="CachingTokenFilter"/>
- </linecontainsregexp>
- <linecontainsregexp negate="true">
- <!-- no way to leverage this in Solr -->
- <regexp pattern="HyphenationCompoundWordTokenFilter"/>
- </linecontainsregexp>
- <linecontainsregexp negate="true">
- <!-- no way to leverage these in Solr (yet) -->
- <regexp pattern="Sink|Tee"/>
- </linecontainsregexp>
- <linecontainsregexp negate="true">
- <!-- Solr already has a different impl for this -->
- <regexp pattern="SynonymTokenFilter"/>
- </linecontainsregexp>
- <linecontainsregexp negate="true">
- <!-- solr and lucene both have one? ? ? ? -->
- <regexp pattern="LengthFilter"/>
- </linecontainsregexp>
- <linecontainsregexp negate="true">
- <!-- solr provides it's own SnowballPorterFilter variant -->
- <regexp pattern="SnowballFilter"/>
- </linecontainsregexp>
- </inputfilterchain>
- </redirector>
- <arg line="${stub.src.dirs}"/>
- </exec>
- </target>
-
- <target name="svn-up">
- <exec executable="${svn.exe}">
- <arg value="update"/>
- </exec>
- </target>
-
+
<property name="analysis-common.res.dir" value="../lucene/analysis/common/src/resources/org/apache/lucene/analysis"/>
<property name="analysis-kuromoji.res.dir" value="../lucene/analysis/kuromoji/src/resources/org/apache/lucene/analysis"/>
<property name="analysis.conf.dest" value="${example}/solr/conf/lang"/>
@@ -748,17 +719,7 @@
tofile="${analysis.conf.dest}/stopwords_tr.txt"/>
</target>
- <!-- TODO: in the future, we don't need to actually put
- jars in the lib/ folders, but can just put in classpath.
- only packaging tasks really need that (and could do it
- under build/ directories) -->
- <target name="clean-jars" description="Clean local jars">
- <delete>
- <fileset dir="." includes="**/*.jar"/>
- </delete>
- </target>
-
- <target name="jar-checksums" depends="clean-jars,resolve">
+ <target name="jar-checksums" depends="resolve">
<jar-checksum-macro srcdir="${common-solr.dir}" dstdir="${common-solr.dir}/licenses"/>
</target>
Modified: lucene/dev/branches/LUCENE-2878/solr/cloud-dev/functions.sh
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/cloud-dev/functions.sh?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/cloud-dev/functions.sh (original)
+++ lucene/dev/branches/LUCENE-2878/solr/cloud-dev/functions.sh Fri Jan 18 18:30:54 2013
@@ -40,7 +40,7 @@ start() {
NUMSHARDS="1"
fi
echo "Instance is running zk, numshards=$NUMSHARDS"
- OPT="-DzkRun -Dbootstrap_confdir=solr/conf -DnumShards=$NUMSHARDS"
+ OPT="-DzkRun -Dbootstrap_conf=true -DnumShards=$NUMSHARDS"
fi
setports $1
cd ../example$1
Modified: lucene/dev/branches/LUCENE-2878/solr/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/common-build.xml?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/common-build.xml (original)
+++ lucene/dev/branches/LUCENE-2878/solr/common-build.xml Fri Jan 18 18:30:54 2013
@@ -25,7 +25,7 @@
<property name="Name" value="Solr" />
<property name="version" value="5.0-SNAPSHOT"/>
- <property name="fullname" value="apache-${ant.project.name}"/>
+ <property name="fullname" value="${ant.project.name}"/>
<property name="fullnamever" value="${fullname}-${version}"/>
<property name="final.name" value="${fullnamever}"/>
@@ -114,7 +114,7 @@
<attribute name="property" default="@{name}.uptodate"/>
<attribute name="classpath.property" default="@{name}.jar"/>
<!-- set jarfile only, if the target jar file has no generic name -->
- <attribute name="jarfile" default="${common-solr.dir}/build/contrib/solr-@{name}/apache-solr-@{name}-${version}.jar"/>
+ <attribute name="jarfile" default="${common-solr.dir}/build/contrib/solr-@{name}/solr-@{name}-${version}.jar"/>
<sequential>
<!--<echo message="Checking '@{jarfile}' against source folder '${common.dir}/contrib/@{name}/src/java'"/>-->
<property name="@{classpath.property}" location="@{jarfile}"/>
@@ -214,13 +214,13 @@
</target>
<target name="check-solr-core-javadocs-uptodate" unless="solr-core-javadocs.uptodate">
- <uptodate property="solr-core-javadocs.uptodate" targetfile="${build.dir}/solr-core/apache-solr-core-${version}-javadoc.jar">
+ <uptodate property="solr-core-javadocs.uptodate" targetfile="${build.dir}/solr-core/solr-core-${version}-javadoc.jar">
<srcfiles dir="${common-solr.dir}/core/src/java" includes="**/*.java"/>
</uptodate>
</target>
<target name="check-solrj-javadocs-uptodate" unless="solrj-javadocs.uptodate">
- <uptodate property="solrj-javadocs.uptodate" targetfile="${build.dir}/solr-solrj/apache-solr-solrj-${version}-javadoc.jar">
+ <uptodate property="solrj-javadocs.uptodate" targetfile="${build.dir}/solr-solrj/solr-solrj-${version}-javadoc.jar">
<srcfiles dir="${common-solr.dir}/solrj/src/java" includes="**/*.java"/>
</uptodate>
</target>
@@ -306,18 +306,8 @@
<target name="dist-maven-src-java"
depends="filter-pom-templates, install-maven-tasks, m2-deploy-solr-parent-pom, dist-maven-common-src-java"/>
- <target name="-validate-maven-dependencies">
- <property name="top.level.dir" location="${common.dir}/.."/>
- <pathconvert property="pom.xml">
- <mapper>
- <chainedmapper>
- <globmapper from="${top.level.dir}*" to="${filtered.pom.templates.dir}*"/>
- <globmapper from="*build.xml" to="*pom.xml"/>
- </chainedmapper>
- </mapper>
- <path location="${ant.file}"/>
- </pathconvert>
- <m2-validate-dependencies pom.xml="${pom.xml}" licenseDirectory="${license.dir}">
+ <target name="-validate-maven-dependencies" depends="-validate-maven-dependencies.init">
+ <m2-validate-dependencies pom.xml="${maven.pom.xml}" licenseDirectory="${license.dir}">
<additional-filters>
<replaceregex pattern="jetty([^/]+)$" replace="jetty" flags="gi" />
<replaceregex pattern="slf4j-([^/]+)$" replace="slf4j" flags="gi" />
@@ -325,8 +315,8 @@
</additional-filters>
<excludes>
<rsel:or>
- <rsel:name name="**/lucene-*.jar" handledirsep="true"/>
- <rsel:name name="**/solr-*.jar" handledirsep="true"/>
+ <rsel:name name="**/lucene-*-${maven.version.glob}.jar" handledirsep="true"/>
+ <rsel:name name="**/solr-*-${maven.version.glob}.jar" handledirsep="true"/>
<!-- TODO: figure out what is going on here with servlet-apis -->
<rsel:name name="**/*servlet*.jar" handledirsep="true"/>
</rsel:or>
@@ -432,4 +422,9 @@
<delete dir="${dest}/web" includes="**/*" failonerror="false"/>
<contrib-crawl target="add-to-war"/>
</target>
+
+ <!-- hack for now to disable *all* Solr tests on Jenkins when "tests.disable-solr" property is set -->
+ <target name="test" unless="tests.disable-solr">
+ <antcall target="common.test" inheritrefs="true" inheritall="true"/>
+ </target>
</project>
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/analysis-extras/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/analysis-extras/ivy.xml?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/analysis-extras/ivy.xml (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/analysis-extras/ivy.xml Fri Jan 18 18:30:54 2013
@@ -20,9 +20,9 @@
<info organisation="org.apache.solr" module="analysis-extras"/>
<dependencies>
<dependency org="com.ibm.icu" name="icu4j" rev="49.1" transitive="false"/>
- <dependency org="org.carrot2" name="morfologik-polish" rev="1.5.3" transitive="false"/>
- <dependency org="org.carrot2" name="morfologik-fsa" rev="1.5.3" transitive="false"/>
- <dependency org="org.carrot2" name="morfologik-stemming" rev="1.5.3" transitive="false"/>
+ <dependency org="org.carrot2" name="morfologik-polish" rev="1.5.5" transitive="false"/>
+ <dependency org="org.carrot2" name="morfologik-fsa" rev="1.5.5" transitive="false"/>
+ <dependency org="org.carrot2" name="morfologik-stemming" rev="1.5.5" transitive="false"/>
<exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
</dependencies>
</ivy-module>
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/clustering/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/clustering/ivy.xml?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/clustering/ivy.xml (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/clustering/ivy.xml Fri Jan 18 18:30:54 2013
@@ -19,13 +19,14 @@
<ivy-module version="2.0">
<info organisation="org.apache.solr" module="clustering"/>
<dependencies>
- <dependency org="org.carrot2" name="carrot2-core" rev="3.5.0" transitive="false"/>
- <dependency org="com.carrotsearch" name="hppc" rev="0.3.3" transitive="false"/>
+ <dependency org="org.carrot2" name="carrot2-core" rev="3.6.2" transitive="false"/>
+ <dependency org="org.carrot2.attributes" name="attributes-binder" rev="1.0.1" transitive="false"/>
+ <dependency org="com.carrotsearch" name="hppc" rev="0.4.1" transitive="false"/>
<dependency org="org.codehaus.jackson" name="jackson-core-asl" rev="1.7.4" transitive="false"/>
<dependency org="org.codehaus.jackson" name="jackson-mapper-asl" rev="1.7.4" transitive="false"/>
- <dependency org="org.apache.mahout" name="mahout-collections" rev="0.3" transitive="false"/>
- <dependency org="org.apache.mahout" name="mahout-math" rev="0.3" transitive="false"/>
- <dependency org="org.simpleframework" name="simple-xml" rev="2.4.1" transitive="false"/>
+ <dependency org="org.apache.mahout" name="mahout-collections" rev="1.0" transitive="false"/>
+ <dependency org="org.apache.mahout" name="mahout-math" rev="0.6" transitive="false"/>
+ <dependency org="org.simpleframework" name="simple-xml" rev="2.6.4" transitive="false"/>
<exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
</dependencies>
</ivy-module>
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/ClusteringComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/ClusteringComponent.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/ClusteringComponent.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/ClusteringComponent.java Fri Jan 18 18:30:54 2013
@@ -187,6 +187,7 @@ public class ClusteringComponent extends
this.initParams = args;
}
+ @Override
public void inform(SolrCore core) {
if (initParams != null) {
log.info("Initializing Clustering Engines");
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngine.java Fri Jan 18 18:30:54 2013
@@ -132,13 +132,10 @@ public class CarrotClusteringEngine exte
try {
resourceStream = resourceLoader.openResource(resourceName);
asBytes = IOUtils.toByteArray(resourceStream);
- } catch (RuntimeException e) {
+ } catch (IOException e) {
log.debug("Resource not found in Solr's config: " + resourceName
+ ". Using the default " + resource + " from Carrot JAR.");
return new IResource[] {};
- } catch (IOException e) {
- log.warn("Could not read Solr resource " + resourceName);
- return new IResource[] {};
} finally {
if (resourceStream != null) Closeables.closeQuietly(resourceStream);
}
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2StemmerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2StemmerFactory.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2StemmerFactory.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2StemmerFactory.java Fri Jan 18 18:30:54 2013
@@ -113,6 +113,7 @@ public class LuceneCarrot2StemmerFactory
this.snowballStemmer = snowballStemmer;
}
+ @Override
public CharSequence stem(CharSequence word) {
snowballStemmer.setCurrent(word.toString());
if (snowballStemmer.stem()) {
@@ -185,6 +186,7 @@ public class LuceneCarrot2StemmerFactory
normalizer = new org.apache.lucene.analysis.ar.ArabicNormalizer();
}
+ @Override
public CharSequence stem(CharSequence word) {
if (word.length() > buffer.length) {
buffer = new char[word.length()];
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2TokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2TokenizerFactory.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2TokenizerFactory.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/clustering/src/java/org/apache/solr/handler/clustering/carrot2/LuceneCarrot2TokenizerFactory.java Fri Jan 18 18:30:54 2013
@@ -115,6 +115,7 @@ public class LuceneCarrot2TokenizerFacto
"org.apache.lucene.analysis.cn.smart.WordTokenFilter", false);
}
+ @Override
public short nextToken() throws IOException {
final boolean hasNextToken = wordTokenFilter.incrementToken();
if (hasNextToken) {
@@ -137,10 +138,12 @@ public class LuceneCarrot2TokenizerFacto
return ITokenizer.TT_EOF;
}
+ @Override
public void setTermBuffer(MutableCharArray array) {
array.reset(term.buffer(), 0, term.length());
}
+ @Override
public void reset(Reader input) {
try {
sentenceTokenizer.setReader(input);
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/carrot2/CarrotClusteringEngineTest.java Fri Jan 18 18:30:54 2013
@@ -116,7 +116,7 @@ public class CarrotClusteringEngineTest
@Test
public void testCarrotStc() throws Exception {
- checkEngine(getClusteringEngine("stc"), 1);
+ checkEngine(getClusteringEngine("stc"), 2);
}
@Test
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/MailEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/MailEntityProcessor.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/MailEntityProcessor.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler-extras/src/java/org/apache/solr/handler/dataimport/MailEntityProcessor.java Fri Jan 18 18:30:54 2013
@@ -323,10 +323,12 @@ public class MailEntityProcessor extends
getTopLevelFolders(mailBox);
}
+ @Override
public boolean hasNext() {
return !folders.isEmpty();
}
+ @Override
public Folder next() {
try {
boolean hasMessages = false;
@@ -370,6 +372,7 @@ public class MailEntityProcessor extends
return null;
}
+ @Override
public void remove() {
throw new UnsupportedOperationException("Its read only mode...");
}
@@ -461,6 +464,7 @@ public class MailEntityProcessor extends
LOG.info("Messages in this batch : " + messagesInCurBatch.length);
}
+ @Override
public boolean hasNext() {
boolean hasMore = current < messagesInCurBatch.length;
if (!hasMore && doBatching
@@ -477,10 +481,12 @@ public class MailEntityProcessor extends
return hasMore;
}
+ @Override
public Message next() {
return hasNext() ? messagesInCurBatch[current++] : null;
}
+ @Override
public void remove() {
throw new UnsupportedOperationException("Its read only mode...");
}
@@ -510,6 +516,7 @@ public class MailEntityProcessor extends
since = date;
}
+ @Override
public SearchTerm getCustomSearch(Folder folder) {
return new ReceivedDateTerm(ComparisonTerm.GE, since);
}
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/build.xml?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/build.xml (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/build.xml Fri Jan 18 18:30:54 2013
@@ -24,7 +24,7 @@
</description>
<!-- the tests have some parallel problems: writability to single copy of dataimport.properties -->
- <property name="tests.jvms" value="1"/>
+ <property name="tests.jvms.override" value="1"/>
<import file="../contrib-build.xml"/>
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCache.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCache.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCache.java Fri Jan 18 18:30:54 2013
@@ -77,6 +77,7 @@ public interface DIHCache extends Iterab
* in key, then insertion, order.
* </p>
*/
+ @Override
public Iterator<Map<String,Object>> iterator();
/**
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java Fri Jan 18 18:30:54 2013
@@ -81,6 +81,7 @@ public class DataImportHandler extends R
super.init(args);
}
+ @Override
@SuppressWarnings("unchecked")
public void inform(SolrCore core) {
try {
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatEvaluator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatEvaluator.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatEvaluator.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DateFormatEvaluator.java Fri Jan 18 18:30:54 2013
@@ -7,9 +7,11 @@ import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Set;
import java.util.TimeZone;
import java.util.WeakHashMap;
@@ -52,13 +54,16 @@ public class DateFormatEvaluator extends
public static final String DEFAULT_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss";
Map<DateFormatCacheKey, SimpleDateFormat> cache = new WeakHashMap<DateFormatCacheKey, SimpleDateFormat>();
Map<String, Locale> availableLocales = new HashMap<String, Locale>();
+ Set<String> availableTimezones = new HashSet<String>();
class DateFormatCacheKey {
- DateFormatCacheKey(Locale l, String df) {
+ DateFormatCacheKey(Locale l, TimeZone tz, String df) {
this.locale = l;
+ this.timezone = tz;
this.dateFormat = df;
}
Locale locale;
+ TimeZone timezone;
String dateFormat;
}
@@ -66,14 +71,27 @@ public class DateFormatEvaluator extends
for (Locale locale : Locale.getAvailableLocales()) {
availableLocales.put(locale.toString(), locale);
}
+ for (String tz : TimeZone.getAvailableIDs()) {
+ availableTimezones.add(tz);
+ }
+ }
+ private SimpleDateFormat getDateFormat(String pattern, TimeZone timezone, Locale locale) {
+ DateFormatCacheKey dfck = new DateFormatCacheKey(locale, timezone, pattern);
+ SimpleDateFormat sdf = cache.get(dfck);
+ if(sdf == null) {
+ sdf = new SimpleDateFormat(pattern, locale);
+ sdf.setTimeZone(timezone);
+ cache.put(dfck, sdf);
+ }
+ return sdf;
}
@Override
public String evaluate(String expression, Context context) {
List<Object> l = parseParams(expression, context.getVariableResolver());
- if (l.size() < 2 || l.size() > 3) {
- throw new DataImportHandlerException(SEVERE, "'formatDate()' must have two or three parameters ");
+ if (l.size() < 2 || l.size() > 4) {
+ throw new DataImportHandlerException(SEVERE, "'formatDate()' must have two, three or four parameters ");
}
Object o = l.get(0);
Object format = l.get(1);
@@ -81,15 +99,13 @@ public class DateFormatEvaluator extends
VariableWrapper wrapper = (VariableWrapper) format;
o = wrapper.resolve();
format = o.toString();
- }
+ }
Locale locale = Locale.ROOT;
- if(l.size()==3) {
+ if(l.size()>2) {
Object localeObj = l.get(2);
String localeStr = null;
if (localeObj instanceof VariableWrapper) {
- VariableWrapper wrapper = (VariableWrapper) localeObj;
- o = wrapper.resolve();
- localeStr = o.toString();
+ localeStr = ((VariableWrapper) localeObj).resolve().toString();
} else {
localeStr = localeObj.toString();
}
@@ -98,13 +114,23 @@ public class DateFormatEvaluator extends
throw new DataImportHandlerException(SEVERE, "Unsupported locale: " + localeStr);
}
}
- String dateFmt = format.toString();
- DateFormatCacheKey dfck = new DateFormatCacheKey(locale, dateFmt);
- SimpleDateFormat sdf = cache.get(dfck);
- if(sdf==null) {
- sdf = new SimpleDateFormat(dateFmt, locale);
- cache.put(dfck, sdf);
+ TimeZone tz = TimeZone.getDefault();
+ if(l.size()==4) {
+ Object tzObj = l.get(3);
+ String tzStr = null;
+ if (tzObj instanceof VariableWrapper) {
+ tzStr = ((VariableWrapper) tzObj).resolve().toString();
+ } else {
+ tzStr = tzObj.toString();
+ }
+ if(availableTimezones.contains(tzStr)) {
+ tz = TimeZone.getTimeZone(tzStr);
+ } else {
+ throw new DataImportHandlerException(SEVERE, "Unsupported Timezone: " + tzStr);
+ }
}
+ String dateFmt = format.toString();
+ SimpleDateFormat fmt = getDateFormat(dateFmt, tz, locale);
Date date = null;
if (o instanceof VariableWrapper) {
VariableWrapper variableWrapper = (VariableWrapper) o;
@@ -114,13 +140,7 @@ public class DateFormatEvaluator extends
} else {
String s = variableval.toString();
try {
- dfck = new DateFormatCacheKey(locale, DEFAULT_DATE_FORMAT);
- sdf = cache.get(dfck);
- if(sdf==null) {
- sdf = new SimpleDateFormat(dfck.dateFormat, dfck.locale);
- cache.put(dfck, sdf);
- }
- date = new SimpleDateFormat(DEFAULT_DATE_FORMAT, locale).parse(s);
+ date = getDateFormat(DEFAULT_DATE_FORMAT, tz, locale).parse(s);
} catch (ParseException exp) {
wrapAndThrow(SEVERE, exp, "Invalid expression for date");
}
@@ -129,15 +149,15 @@ public class DateFormatEvaluator extends
String datemathfmt = o.toString();
datemathfmt = datemathfmt.replaceAll("NOW", "");
try {
- date = getDateMathParser(locale).parseMath(datemathfmt);
+ date = getDateMathParser(locale, tz).parseMath(datemathfmt);
} catch (ParseException e) {
wrapAndThrow(SEVERE, e, "Invalid expression for date");
}
}
- return sdf.format(date);
+ return fmt.format(date);
}
- static DateMathParser getDateMathParser(Locale l) {
- return new DateMathParser(TimeZone.getDefault(), l) {
+ static DateMathParser getDateMathParser(Locale l, TimeZone tz) {
+ return new DateMathParser(tz, l) {
@Override
public Date getNow() {
return new Date();
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java Fri Jan 18 18:30:54 2013
@@ -68,7 +68,6 @@ public class DocBuilder {
Map<String, Object> session = new HashMap<String, Object>();
static final ThreadLocal<DocBuilder> INSTANCE = new ThreadLocal<DocBuilder>();
- //private Map<String, Object> functionsNamespace;
private Map<String, Object> persistedProperties;
private DIHProperties propWriter;
@@ -640,11 +639,20 @@ public class DocBuilder {
if (field != null) {
for (EntityField f : field) {
String name = f.getName();
+ boolean multiValued = f.isMultiValued();
+ boolean toWrite = f.isToWrite();
if(f.isDynamicName()){
name = vr.replaceTokens(name);
+ SchemaField schemaField = dataImporter.getSchemaField(name);
+ if(schemaField == null) {
+ toWrite = false;
+ } else {
+ multiValued = schemaField.multiValued();
+ toWrite = true;
+ }
}
- if (f.isToWrite()) {
- addFieldToDoc(entry.getValue(), name, f.getBoost(), f.isMultiValued(), doc);
+ if (toWrite) {
+ addFieldToDoc(entry.getValue(), name, f.getBoost(), multiValued, doc);
}
}
}
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileDataSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileDataSource.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileDataSource.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileDataSource.java Fri Jan 18 18:30:54 2013
@@ -92,22 +92,35 @@ public class FileDataSource extends Data
static File getFile(String basePath, String query) {
try {
- File file0 = new File(query);
- File file = file0;
+ File file = new File(query);
- if (!file.isAbsolute())
- file = new File(basePath + query);
+ // If it's not an absolute path, try relative from basePath.
+ if (!file.isAbsolute()) {
+ // Resolve and correct basePath.
+ File basePathFile;
+ if (basePath == null) {
+ basePathFile = new File(".").getAbsoluteFile();
+ LOG.warn("FileDataSource.basePath is empty. " +
+ "Resolving to: " + basePathFile.getAbsolutePath());
+ } else {
+ basePathFile = new File(basePath);
+ if (!basePathFile.isAbsolute()) {
+ basePathFile = basePathFile.getAbsoluteFile();
+ LOG.warn("FileDataSource.basePath is not absolute. Resolving to: "
+ + basePathFile.getAbsolutePath());
+ }
+ }
+
+ file = new File(basePathFile, query).getAbsoluteFile();
+ }
if (file.isFile() && file.canRead()) {
- LOG.debug("Accessing File: " + file.toString());
+ LOG.debug("Accessing File: " + file.getAbsolutePath());
return file;
- } else if (file != file0)
- if (file0.isFile() && file0.canRead()) {
- LOG.debug("Accessing File0: " + file0.toString());
- return file0;
- }
-
- throw new FileNotFoundException("Could not find file: " + query);
+ } else {
+ throw new FileNotFoundException("Could not find file: " + query +
+ " (resolved to: " + file.getAbsolutePath());
+ }
} catch (FileNotFoundException e) {
throw new RuntimeException(e);
}
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileListEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileListEntityProcessor.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileListEntityProcessor.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileListEntityProcessor.java Fri Jan 18 18:30:54 2013
@@ -157,7 +157,7 @@ public class FileListEntityProcessor ext
String expr = null;
expr = m.group(1).replaceAll("NOW", "");
try {
- return DateFormatEvaluator.getDateMathParser(Locale.ROOT).parseMath(expr);
+ return DateFormatEvaluator.getDateMathParser(Locale.ROOT, TimeZone.getDefault()).parseMath(expr);
} catch (ParseException exp) {
throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
"Invalid expression for date", exp);
@@ -225,6 +225,7 @@ public class FileListEntityProcessor ext
// Rather we make use of the fileDetails array which is populated as
// a side affect of the accept method.
dir.list(new FilenameFilter() {
+ @Override
public boolean accept(File dir, String name) {
File fileObj = new File(dir, name);
if (fileObj.isDirectory()) {
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/JdbcDataSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/JdbcDataSource.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/JdbcDataSource.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/JdbcDataSource.java Fri Jan 18 18:30:54 2013
@@ -125,6 +125,7 @@ public class JdbcDataSource extends
}
return factory = new Callable<Connection>() {
+ @Override
public Connection call() throws Exception {
LOG.info("Creating a connection for entity "
+ context.getEntityAttribute(DataImporter.NAME) + " with URL: "
@@ -257,14 +258,17 @@ public class JdbcDataSource extends
}
rSetIterator = new Iterator<Map<String, Object>>() {
+ @Override
public boolean hasNext() {
return hasnext();
}
+ @Override
public Map<String, Object> next() {
return getARow();
}
+ @Override
public void remove() {/* do nothing */
}
};
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java Fri Jan 18 18:30:54 2013
@@ -84,12 +84,7 @@ public class SimplePropertiesWriter exte
} else {
filename = "dataimport.properties";
}
- if(params.get(DIRECTORY) != null) {
- configDir = params.get(DIRECTORY);
- } else {
- SolrCore core = dataImporter.getCore();
- configDir = (core == null ? "." : core.getResourceLoader().getConfigDir());
- }
+ findDirectory(dataImporter, params);
if(params.get(LOCALE) != null) {
String localeStr = params.get(LOCALE);
for (Locale l : Locale.getAvailableLocales()) {
@@ -109,6 +104,14 @@ public class SimplePropertiesWriter exte
} else {
dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", locale);
}
+ }
+ protected void findDirectory(DataImporter dataImporter, Map<String, String> params) {
+ if(params.get(DIRECTORY) != null) {
+ configDir = params.get(DIRECTORY);
+ } else {
+ SolrCore core = dataImporter.getCore();
+ configDir = (core == null ? "." : core.getResourceLoader().getConfigDir());
+ }
}
private File getPersistFile() {
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrQueryEscapingEvaluator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrQueryEscapingEvaluator.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrQueryEscapingEvaluator.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrQueryEscapingEvaluator.java Fri Jan 18 18:30:54 2013
@@ -24,6 +24,7 @@ import org.apache.solr.client.solrj.util
*/
public class SolrQueryEscapingEvaluator extends Evaluator {
+ @Override
public String evaluate(String expression, Context context) {
List<Object> l = parseParams(expression, context.getVariableResolver());
if (l.size() != 1) {
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java Fri Jan 18 18:30:54 2013
@@ -63,6 +63,7 @@ public class VariableResolver {
}
public static final String FUNCTIONS_NAMESPACE = "dataimporter.functions.";
+ public static final String FUNCTIONS_NAMESPACE_SHORT = "dih.functions.";
public VariableResolver() {
rootNamespace = new HashMap<String,Object>();
@@ -95,7 +96,11 @@ public class VariableResolver {
r = currentLevel.get(nameParts[nameParts.length - 1]);
if (r == null && name.startsWith(FUNCTIONS_NAMESPACE)
&& name.length() > FUNCTIONS_NAMESPACE.length()) {
- return resolveEvaluator(name);
+ return resolveEvaluator(FUNCTIONS_NAMESPACE, name);
+ }
+ if (r == null && name.startsWith(FUNCTIONS_NAMESPACE_SHORT)
+ && name.length() > FUNCTIONS_NAMESPACE_SHORT.length()) {
+ return resolveEvaluator(FUNCTIONS_NAMESPACE_SHORT, name);
}
if (r == null) {
r = System.getProperty(name);
@@ -104,12 +109,12 @@ public class VariableResolver {
return r == null ? "" : r;
}
- private Object resolveEvaluator(String name) {
+ private Object resolveEvaluator(String namespace, String name) {
if (evaluators == null) {
return "";
}
Matcher m = EVALUATOR_FORMAT_PATTERN.matcher(name
- .substring(FUNCTIONS_NAMESPACE.length()));
+ .substring(namespace.length()));
if (m.find()) {
String fname = m.group(1);
Evaluator evaluator = evaluators.get(fname);
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathEntityProcessor.java Fri Jan 18 18:30:54 2013
@@ -316,6 +316,7 @@ public class XPathEntityProcessor extend
} else {
try {
xpathReader.streamRecords(data, new XPathRecordReader.Handler() {
+ @Override
@SuppressWarnings("unchecked")
public void handle(Map<String, Object> record, String xpath) {
rows.add(readRow(record, xpath));
@@ -425,6 +426,7 @@ public class XPathEntityProcessor extend
public void run() {
try {
xpathReader.streamRecords(data, new XPathRecordReader.Handler() {
+ @Override
@SuppressWarnings("unchecked")
public void handle(Map<String, Object> record, String xpath) {
if (isEnd.get()) {
@@ -475,10 +477,12 @@ public class XPathEntityProcessor extend
private Map<String, Object> lastRow;
int count = 0;
+ @Override
public boolean hasNext() {
return !isEnd.get();
}
+ @Override
public Map<String, Object> next() {
Map<String, Object> row;
@@ -514,6 +518,7 @@ public class XPathEntityProcessor extend
return lastRow = row;
}
+ @Override
public void remove() {
/*no op*/
}
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathRecordReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathRecordReader.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathRecordReader.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/XPathRecordReader.java Fri Jan 18 18:30:54 2013
@@ -164,6 +164,7 @@ public class XPathRecordReader {
public List<Map<String, Object>> getAllRecords(Reader r) {
final List<Map<String, Object>> results = new ArrayList<Map<String, Object>>();
streamRecords(r, new Handler() {
+ @Override
public void handle(Map<String, Object> record, String s) {
results.add(record);
}
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java Fri Jan 18 18:30:54 2013
@@ -42,12 +42,16 @@ public class ZKPropertiesWriter extends
@Override
public void init(DataImporter dataImporter, Map<String, String> params) {
- super.init(dataImporter, params);
+ super.init(dataImporter, params);
+ zkClient = dataImporter.getCore().getCoreDescriptor().getCoreContainer()
+ .getZkController().getZkClient();
+ }
+
+ @Override
+ protected void findDirectory(DataImporter dataImporter, Map<String, String> params) {
String collection = dataImporter.getCore().getCoreDescriptor()
.getCloudDescriptor().getCollectionName();
path = "/configs/" + collection + "/" + filename;
- zkClient = dataImporter.getCore().getCoreDescriptor().getCoreContainer()
- .getZkController().getZkClient();
}
@Override
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-schema.xml?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-schema.xml (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-schema.xml Fri Jan 18 18:30:54 2013
@@ -41,6 +41,8 @@
<field name="COUNTRY_NAME" type="text" indexed="true" stored="true" multiValued="true" />
<field name="SPORT_NAME" type="text" indexed="true" stored="true" multiValued="true" />
<field name="DO_NOT_INDEX" type="ignored" />
+
+ <field name="_version_" type="tlong" indexed="true" stored="true" multiValued="false"/>
<dynamicField name="*_i" type="tint" indexed="true" stored="true"/>
<dynamicField name="*_s" type="string" indexed="true" stored="true"/>
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHCacheTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHCacheTestCase.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHCacheTestCase.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHCacheTestCase.java Fri Jan 18 18:30:54 2013
@@ -81,6 +81,7 @@ public class AbstractDIHCacheTestCase {
this.data = data;
}
+ @Override
@SuppressWarnings("unchecked")
public int compareTo(ControlData cd) {
Comparable c1 = (Comparable) data[0];
@@ -88,6 +89,7 @@ public class AbstractDIHCacheTestCase {
return c1.compareTo(c2);
}
+ @Override
public Iterator<Object> iterator() {
return Arrays.asList(data).iterator();
}
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHJdbcTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHJdbcTestCase.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHJdbcTestCase.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHJdbcTestCase.java Fri Jan 18 18:30:54 2013
@@ -58,7 +58,9 @@ public abstract class AbstractDIHJdbcTes
try {
Class.forName("org.hsqldb.jdbcDriver").newInstance();
String oldProp = System.getProperty("derby.stream.error.field");
- System.setProperty("derby.stream.error.field", "DerbyUtil.DEV_NULL");
+ System
+ .setProperty("derby.stream.error.field",
+ "org.apache.solr.handler.dataimport.AbstractDIHJdbcTestCase$DerbyUtil.DEV_NULL");
Class.forName("org.apache.derby.jdbc.EmbeddedDriver").newInstance();
if (oldProp != null) {
System.setProperty("derby.stream.error.field", oldProp);
@@ -204,6 +206,7 @@ public abstract class AbstractDIHJdbcTes
public static class DerbyUtil {
public static final OutputStream DEV_NULL = new OutputStream() {
+ @Override
public void write(int b) {}
};
}
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractSqlEntityProcessorTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractSqlEntityProcessorTestCase.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractSqlEntityProcessorTestCase.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractSqlEntityProcessorTestCase.java Fri Jan 18 18:30:54 2013
@@ -1,17 +1,23 @@
package org.apache.solr.handler.dataimport;
+import java.io.File;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Timestamp;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Locale;
+import java.util.Map;
import java.util.Set;
import org.junit.After;
+import org.junit.Before;
import junit.framework.Assert;
@@ -42,8 +48,18 @@ public abstract class AbstractSqlEntityP
protected boolean sportsCached;
protected String rootTransformerName;
protected boolean countryTransformer;
- protected boolean sportsTransformer;
-
+ protected boolean sportsTransformer;
+ protected String fileLocation;
+ protected String fileName;
+
+ @Before
+ public void beforeSqlEntitiyProcessorTestCase() throws Exception {
+ File tmpdir = File.createTempFile("test", "tmp", TEMP_DIR);
+ tmpdir.delete();
+ tmpdir.mkdir();
+ fileLocation = tmpdir.getPath();
+ fileName = "the.properties";
+ }
@After
public void afterSqlEntitiyProcessorTestCase() {
useSimpleCaches = false;
@@ -55,6 +71,30 @@ public abstract class AbstractSqlEntityP
countryTransformer = false;
sportsTransformer = false;
underlyingDataModified = false;
+
+ //If an Assume was tripped while setting up the test,
+ //the file might not ever have been created...
+ if(fileLocation!=null) {
+ new File(fileLocation + File.separatorChar + fileName).delete();
+ new File(fileLocation).delete();
+ }
+ }
+
+ protected void logPropertiesFile() {
+ Map<String,String> init = new HashMap<String,String>();
+ init.put("filename", fileName);
+ init.put("directory", fileLocation);
+ SimplePropertiesWriter spw = new SimplePropertiesWriter();
+ spw.init(new DataImporter(), init);
+ Map<String,Object> props = spw.readIndexerProperties();
+ if(props!=null) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("\ndataimporter.properties: \n");
+ for(Map.Entry<String,Object> entry : props.entrySet()) {
+ sb.append(" > key=" + entry.getKey() + " / value=" + entry.getValue() + "\n");
+ }
+ log.debug(sb.toString());
+ }
}
protected abstract String deltaQueriesCountryTable();
@@ -379,6 +419,9 @@ public abstract class AbstractSqlEntityP
// One second in the future ensures a change time after the last import (DIH
// uses second precision only)
Timestamp theTime = new Timestamp(System.currentTimeMillis() + 1000);
+ log.debug("PEOPLE UPDATE USING TIMESTAMP: "
+ + new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ", Locale.ROOT)
+ .format(theTime));
try {
conn = newConnection();
change = conn
@@ -447,6 +490,9 @@ public abstract class AbstractSqlEntityP
// One second in the future ensures a change time after the last import (DIH
// uses second precision only)
Timestamp theTime = new Timestamp(System.currentTimeMillis() + 1000);
+ log.debug("COUNTRY UPDATE USING TIMESTAMP: "
+ + new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ", Locale.ROOT)
+ .format(theTime));
try {
conn = newConnection();
change = conn
@@ -480,6 +526,30 @@ public abstract class AbstractSqlEntityP
public Integer[] changedKeys;
public Integer[] deletedKeys;
public Integer[] addedKeys;
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ if(changedKeys!=null) {
+ sb.append("changes: ");
+ for(int i : changedKeys) {
+ sb.append(i).append(" ");
+ }
+ }
+ if(deletedKeys!=null) {
+ sb.append("deletes: ");
+ for(int i : deletedKeys) {
+ sb.append(i).append(" ");
+ }
+ }
+ if(addedKeys!=null) {
+ sb.append("adds: ");
+ for(int i : addedKeys) {
+ sb.append(i).append(" ");
+ }
+ }
+ return sb.toString();
+ }
}
@Override
@@ -493,7 +563,8 @@ public abstract class AbstractSqlEntityP
throw new AssertionError("Invalid database to use: " + dbToUse);
}
StringBuilder sb = new StringBuilder();
- sb.append("<dataConfig> \n");
+ sb.append("\n<dataConfig> \n");
+ sb.append("<propertyWriter type=''SimplePropertiesWriter'' directory=''" + fileLocation + "'' filename=''" + fileName + "'' />\n");
sb.append("<dataSource name=''hsqldb'' driver=''org.hsqldb.jdbcDriver'' url=''jdbc:hsqldb:mem:.'' /> \n");
sb.append("<dataSource name=''derby'' driver=''org.apache.derby.jdbc.EmbeddedDriver'' url=''jdbc:derby:memory:derbyDB;'' /> \n");
sb.append("<document name=''TestSqlEntityProcessor''> \n");
@@ -580,7 +651,9 @@ public abstract class AbstractSqlEntityP
s.executeUpdate("create table countries(code varchar(3) not null primary key, country_name varchar(50), deleted char(1) default 'N', last_modified timestamp not null)");
s.executeUpdate("create table people(id int not null primary key, name varchar(50), country_code char(2), deleted char(1) default 'N', last_modified timestamp not null)");
s.executeUpdate("create table people_sports(id int not null primary key, person_id int, sport_name varchar(50), deleted char(1) default 'N', last_modified timestamp not null)");
-
+ log.debug("INSERTING DB DATA USING TIMESTAMP: "
+ + new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ", Locale.ROOT)
+ .format(theTime));
ps = conn
.prepareStatement("insert into countries (code, country_name, last_modified) values (?,?,?)");
for (String[] country : countries) {
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/MockInitialContextFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/MockInitialContextFactory.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/MockInitialContextFactory.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/MockInitialContextFactory.java Fri Jan 18 18:30:54 2013
@@ -39,6 +39,7 @@ public class MockInitialContextFactory i
try {
EasyMock.expect(context.lookup((String) EasyMock.anyObject())).andAnswer(
new IAnswer<Object>() {
+ @Override
public Object answer() throws Throwable {
return objects.get(EasyMock.getCurrentArguments()[0]);
}
@@ -51,6 +52,7 @@ public class MockInitialContextFactory i
mockControl.replay();
}
+ @Override
@SuppressWarnings("unchecked")
public javax.naming.Context getInitialContext(Hashtable env) {
return context;
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java Fri Jan 18 18:30:54 2013
@@ -24,7 +24,7 @@ import java.text.SimpleDateFormat;
import java.util.*;
/**
- * <p> Test for EvaluatorBag </p>
+ * <p> Test for Evaluators </p>
*
*
* @since solr 1.3
@@ -107,7 +107,13 @@ public class TestBuiltInEvaluators exten
org.junit.Assert.assertEquals("c\\:t", s);
}
-
+
+ private Date getNow() {
+ Calendar calendar = new GregorianCalendar(TimeZone.getTimeZone("GMT"),
+ Locale.ROOT);
+ calendar.add(Calendar.DAY_OF_YEAR, -2);
+ return calendar.getTime();
+ }
@Test
public void testDateFormatEvaluator() {
@@ -116,23 +122,26 @@ public class TestBuiltInEvaluators exten
Context.FULL_DUMP, Collections.<String,Object> emptyMap(), null, null);
String currentLocale = Locale.getDefault().toString();
{
- Calendar calendar = new GregorianCalendar(TimeZone.getTimeZone("GMT"),
- Locale.ROOT);
- calendar.add(Calendar.DAY_OF_YEAR, -2);
- Date d = calendar.getTime();
{
- String sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm", Locale.ROOT).format(d);
- String dfe = dateFormatEval.evaluate("'NOW-2DAYS','yyyy-MM-dd HH:mm'", context);
+ SimpleDateFormat sdfDate = new SimpleDateFormat("yyyy-MM-dd HH", Locale.ROOT);
+ String sdf = sdfDate.format(getNow());
+ String dfe = dateFormatEval.evaluate("'NOW-2DAYS','yyyy-MM-dd HH'", context);
assertEquals(sdf,dfe);
}
{
- String sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm", Locale.getDefault()).format(d);
- String dfe = dateFormatEval.evaluate("'NOW-2DAYS','yyyy-MM-dd HH:mm','"+ currentLocale + "'", context);
+ SimpleDateFormat sdfDate = new SimpleDateFormat("yyyy-MM-dd HH", Locale.getDefault());
+ String sdf = sdfDate.format(getNow());
+ String dfe = dateFormatEval.evaluate("'NOW-2DAYS','yyyy-MM-dd HH','"+ currentLocale + "'", context);
assertEquals(sdf,dfe);
+ for(String tz : TimeZone.getAvailableIDs()) {
+ sdfDate.setTimeZone(TimeZone.getTimeZone(tz));
+ sdf = sdfDate.format(getNow());
+ dfe = dateFormatEval.evaluate("'NOW-2DAYS','yyyy-MM-dd HH','" + currentLocale + "','" + tz + "'", context);
+ assertEquals(sdf,dfe);
+ }
}
}
- Date d = new Date();
-
+ Date d = new Date();
Map<String,Object> map = new HashMap<String,Object>();
map.put("key", d);
resolver.addNamespace("A", map);
@@ -144,6 +153,15 @@ public class TestBuiltInEvaluators exten
new SimpleDateFormat("yyyy-MM-dd HH:mm", Locale.getDefault()).format(d),
dateFormatEval.evaluate("A.key, 'yyyy-MM-dd HH:mm','" + currentLocale
+ "'", context));
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm", Locale.getDefault());
+ for(String tz : TimeZone.getAvailableIDs()) {
+ sdf.setTimeZone(TimeZone.getTimeZone(tz));
+ assertEquals(
+ sdf.format(d),
+ dateFormatEval.evaluate("A.key, 'yyyy-MM-dd HH:mm','" + currentLocale + "', '" + tz + "'", context));
+
+ }
+
}
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestClobTransformer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestClobTransformer.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestClobTransformer.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestClobTransformer.java Fri Jan 18 18:30:54 2013
@@ -46,6 +46,7 @@ public class TestClobTransformer extends
Transformer t = new ClobTransformer();
Map<String, Object> row = new HashMap<String, Object>();
Clob clob = (Clob) Proxy.newProxyInstance(this.getClass().getClassLoader(), new Class[]{Clob.class}, new InvocationHandler() {
+ @Override
public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
if (method.getName().equals("getCharacterStream")) {
return new StringReader("hello!");
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java Fri Jan 18 18:30:54 2013
@@ -238,7 +238,7 @@ public class TestDocBuilder2 extends Abs
}
@Test
- @Ignore("Known Locale/TZ problems: see https://issues.apache.org/jira/browse/SOLR-1916")
+ @Ignore("Fix Me. See SOLR-4103.")
public void testFileListEntityProcessor_lastIndexTime() throws Exception {
File tmpdir = File.createTempFile("test", "tmp", TEMP_DIR);
tmpdir.delete();
@@ -285,6 +285,7 @@ public class TestDocBuilder2 extends Abs
public static class StartEventListener implements EventListener {
public static boolean executed = false;
+ @Override
public void onEvent(Context ctx) {
executed = true;
}
@@ -293,6 +294,7 @@ public class TestDocBuilder2 extends Abs
public static class EndEventListener implements EventListener {
public static boolean executed = false;
+ @Override
public void onEvent(Context ctx) {
executed = true;
}
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java Fri Jan 18 18:30:54 2013
@@ -132,7 +132,6 @@ public class TestFileListEntityProcessor
}
@Test
- @Ignore("Known Locale/TZ problems: see https://issues.apache.org/jira/browse/SOLR-1916")
public void testNTOT() throws IOException {
File tmpdir = File.createTempFile("test", "tmp", TEMP_DIR);
tmpdir.delete();
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNestedChildren.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNestedChildren.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNestedChildren.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNestedChildren.java Fri Jan 18 18:30:54 2013
@@ -40,7 +40,7 @@ public class TestNestedChildren extends
sb.append(" <entity name=\"SECOND\" processor=\"SqlEntityProcessor\" dataSource=\"derby\" ");
sb.append(" query=\"select 1 as id, 2 as SECOND_ID, 'BEEF' as SECOND_S from sysibm.sysdummy1 WHERE 1=${FIRST.ID}\" >\n");
sb.append(" <field column=\"SECOND_S\" name=\"second_s\" /> \n");
- sb.append(" <entity name=\"SECOND\" processor=\"SqlEntityProcessor\" dataSource=\"derby\" ");
+ sb.append(" <entity name=\"THIRD\" processor=\"SqlEntityProcessor\" dataSource=\"derby\" ");
sb.append(" query=\"select 1 as id, 'CHICKEN' as THIRD_S from sysibm.sysdummy1 WHERE 2=${SECOND.SECOND_ID}\" >\n");
sb.append(" <field column=\"THIRD_S\" name=\"third_s\" /> \n");
sb.append(" </entity>\n");
Modified: lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNonWritablePersistFile.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNonWritablePersistFile.java?rev=1435287&r1=1435286&r2=1435287&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNonWritablePersistFile.java (original)
+++ lucene/dev/branches/LUCENE-2878/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNonWritablePersistFile.java Fri Jan 18 18:30:54 2013
@@ -16,6 +16,8 @@
*/
package org.apache.solr.handler.dataimport;
+import org.apache.commons.io.FileUtils;
+import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -46,12 +48,22 @@ public class TestNonWritablePersistFile
" </entity>\n" +
" </document>\n" +
"</dataConfig>\n";
+ private static String tmpSolrHome;
@BeforeClass
- public static void beforeClass() throws Exception {
- initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
+ public static void createTempSolrHomeAndCore() throws Exception {
+ createTempDir();
+ tmpSolrHome = TEMP_DIR + File.separator + TestNonWritablePersistFile.class.getSimpleName() + System.currentTimeMillis();
+ FileUtils.copyDirectory(getFile("dih/solr"), new File(tmpSolrHome).getAbsoluteFile());
+ initCore("dataimport-solrconfig.xml", "dataimport-schema.xml",
+ new File(tmpSolrHome).getAbsolutePath());
}
+ @AfterClass
+ public static void destroyTempSolrHomeAndCore() throws Exception {
+ FileUtils.deleteDirectory(new File(tmpSolrHome).getAbsoluteFile());
+ }
+
@Test
@SuppressWarnings("unchecked")
public void testNonWritablePersistFile() throws Exception {