You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2013/12/24 14:39:24 UTC
svn commit: r1553272 [1/10] - in
/lucene/dev/branches/lucene5376/lucene/server: ./ plugins/
plugins/BinaryDocument/ plugins/BinaryDocument/src/
plugins/BinaryDocument/src/java/ plugins/BinaryDocument/src/java/org/
plugins/BinaryDocument/src/java/org/ap...
Author: rmuir
Date: Tue Dec 24 13:39:22 2013
New Revision: 1553272
URL: http://svn.apache.org/r1553272
Log:
LUCENE-5376: mike's tgz as-is to branch
Added:
lucene/dev/branches/lucene5376/lucene/server/
lucene/dev/branches/lucene5376/lucene/server/.classpath (with props)
lucene/dev/branches/lucene5376/lucene/server/.hgignore
lucene/dev/branches/lucene5376/lucene/server/.project (with props)
lucene/dev/branches/lucene5376/lucene/server/README.txt (with props)
lucene/dev/branches/lucene5376/lucene/server/build.py (with props)
lucene/dev/branches/lucene5376/lucene/server/build.xml (with props)
lucene/dev/branches/lucene5376/lucene/server/ivy-settings.xml (with props)
lucene/dev/branches/lucene5376/lucene/server/ivy.xml (with props)
lucene/dev/branches/lucene5376/lucene/server/plugins/
lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/
lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/build.py (with props)
lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/build.xml (with props)
lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/dist.py (with props)
lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/ivy-settings.xml (with props)
lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/ivy.xml (with props)
lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/
lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/java/
lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/java/org/
lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/java/org/apache/
lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/java/org/apache/lucene/
lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/java/org/apache/lucene/server/
lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/java/org/apache/lucene/server/handlers/
lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/java/org/apache/lucene/server/handlers/CrackDocumentHandler.java (with props)
lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/java/org/apache/lucene/server/plugins/
lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/java/org/apache/lucene/server/plugins/BinaryDocumentPlugin.java (with props)
lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/resources/
lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/resources/lucene-server-plugin.properties (with props)
lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/test/
lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/test-resources/
lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/test-resources/test.docx (with props)
lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/test/org/
lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/test/org/apache/
lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/test/org/apache/lucene/
lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/test/org/apache/lucene/server/
lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/test/org/apache/lucene/server/TestBinaryDocuments.java (with props)
lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/test.py (with props)
lucene/dev/branches/lucene5376/lucene/server/src/
lucene/dev/branches/lucene5376/lucene/server/src/java/
lucene/dev/branches/lucene5376/lucene/server/src/java/org/
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/BlendedComparator.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/BlockingThreadPoolExecutor.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/Constants.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/DirectoryFactory.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/FieldDef.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/FinishRequest.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/FromFileTermFreqIterator.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/GenFileUtil.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/GlobalState.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/IndexState.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/MultiFacetsAccumulator.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/MyIndexSearcher.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/NamedThreadFactory.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/Pipe.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/PreHandle.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/RecencyBlendedFieldComparatorSource.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/SVJSONPassageFormatter.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/Server.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/StringLiveFieldValues.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/TopFacetsCache.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/WholeMVJSONPassageFormatter.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/AddDocumentHandler.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/AddDocumentsHandler.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/AnalysisHandler.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/BuildSuggestHandler.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/BulkAddDocumentHandler.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/BulkAddDocumentsHandler.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/BulkUpdateDocumentHandler.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/BulkUpdateDocumentsHandler.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/CommitHandler.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/CreateIndexHandler.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/CreateSnapshotHandler.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/DeleteAllDocumentsHandler.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/DeleteDocumentsHandler.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/DeleteIndexHandler.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/DocHandler.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/Handler.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/LiveSettingsHandler.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/LiveValuesHandler.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/RegisterFieldHandler.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/ReleaseSnapshotHandler.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/SearchHandler.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/SettingsHandler.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/ShutdownHandler.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/StartIndexHandler.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/StatsHandler.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/StopIndexHandler.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/SuggestLookupHandler.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/UpdateDocumentHandler.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/http/
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/http/HttpStaticFileServerHandler.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/AnyType.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/BooleanType.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/EnumType.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/FloatType.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/IntType.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/ListType.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/LongType.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/NoType.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/OrType.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/Param.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/PolyType.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/Request.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/RequestFailedException.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/StringType.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/StructType.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/Type.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/WrapType.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/plugins/
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/plugins/Plugin.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/python/
lucene/dev/branches/lucene5376/lucene/server/src/python/buildDocs.py (with props)
lucene/dev/branches/lucene5376/lucene/server/src/python/loadTest2.py (with props)
lucene/dev/branches/lucene5376/lucene/server/src/python/post.py (with props)
lucene/dev/branches/lucene5376/lucene/server/src/python/pp.py (with props)
lucene/dev/branches/lucene5376/lucene/server/src/python/scratchBuild.py (with props)
lucene/dev/branches/lucene5376/lucene/server/src/python/tests.py (with props)
lucene/dev/branches/lucene5376/lucene/server/src/python/util.py (with props)
lucene/dev/branches/lucene5376/lucene/server/src/scripts/
lucene/dev/branches/lucene5376/lucene/server/src/scripts/post.py (with props)
lucene/dev/branches/lucene5376/lucene/server/src/test/
lucene/dev/branches/lucene5376/lucene/server/src/test/org/
lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/
lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/
lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/
lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/MockPlugin-hello.txt (with props)
lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/MockPlugin-lucene-server-plugin.properties (with props)
lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/MockPlugin.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/ServerBaseTestCase.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TermFreqPayload.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestAddDocuments.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestAnalysis.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestBlockJoinQuery.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestBooleanFieldType.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestFacets.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestGrouping.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestHighlight.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestIndexing.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestLiveValues.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestNumericFields.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestPlugins.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestSearch.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestServer.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestSettings.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestSnapshots.java (with props)
lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestSuggest.java (with props)
Added: lucene/dev/branches/lucene5376/lucene/server/.classpath
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/.classpath?rev=1553272&view=auto
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/.classpath (added)
+++ lucene/dev/branches/lucene5376/lucene/server/.classpath Tue Dec 24 13:39:22 2013
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+ <classpathentry kind="src" path="src/java"/>
+ <classpathentry kind="src" path="src/test"/>
+ <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+ <classpathentry kind="con" path="org.apache.ivyde.eclipse.cpcontainer.IVYDE_CONTAINER/?project=luceneserver&ivyXmlPath=ivy.xml&confs=*"/>
+ <classpathentry kind="output" path="bin"/>
+</classpath>
Added: lucene/dev/branches/lucene5376/lucene/server/.hgignore
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/.hgignore?rev=1553272&view=auto
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/.hgignore (added)
+++ lucene/dev/branches/lucene5376/lucene/server/.hgignore Tue Dec 24 13:39:22 2013
@@ -0,0 +1,2 @@
+build
+bin
Added: lucene/dev/branches/lucene5376/lucene/server/.project
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/.project?rev=1553272&view=auto
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/.project (added)
+++ lucene/dev/branches/lucene5376/lucene/server/.project Tue Dec 24 13:39:22 2013
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+ <name>luceneserver</name>
+ <comment></comment>
+ <projects>
+ </projects>
+ <buildSpec>
+ <buildCommand>
+ <name>org.eclipse.jdt.core.javabuilder</name>
+ <arguments>
+ </arguments>
+ </buildCommand>
+ </buildSpec>
+ <natures>
+ <nature>org.eclipse.jdt.core.javanature</nature>
+ <nature>org.apache.ivyde.eclipse.ivynature</nature>
+ </natures>
+</projectDescription>
Added: lucene/dev/branches/lucene5376/lucene/server/README.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/README.txt?rev=1553272&view=auto
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/README.txt (added)
+++ lucene/dev/branches/lucene5376/lucene/server/README.txt Tue Dec 24 13:39:22 2013
@@ -0,0 +1,7 @@
+Simple server wrapping Lucene 4.x.
+
+To run all tests:
+
+ python3 -u build.py test
+
+
Added: lucene/dev/branches/lucene5376/lucene/server/build.py
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/build.py?rev=1553272&view=auto
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/build.py (added)
+++ lucene/dev/branches/lucene5376/lucene/server/build.py Tue Dec 24 13:39:22 2013
@@ -0,0 +1,70 @@
+import zipfile
+import shutil
+import time
+import types
+import sys
+import os
+
+# TODO
+# - package should have README
+# - can i send randomized runner output to stdout?
+# - windows
+
+sys.path.insert(0, 'src/python')
+import util
+
+deps = [
+ ('io.netty', 'netty', '3.6.5.Final'),
+ # For Base64 encode/decode:
+ ('commons-codec', 'commons-codec', '1.7'),
+ ('org.apache.lucene', 'lucene-queryparser', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-misc', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-queries', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-highlighter', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-suggest', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-grouping', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-analyzers-common', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-analyzers-icu', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-facet', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-core', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-codecs', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-join', util.LUCENE_VERSION),
+ ('com.ibm.icu', 'icu4j', '49.1'),
+ ('net.minidev', 'json-smart', '1.1.1'),
+ ('org.codehaus.jackson', 'jackson-core-asl', '1.9.12'),
+ ('org.codehaus.jackson', 'jackson-mapper-asl', '1.9.12'),
+ ]
+
+compileTestDeps = [
+ 'build/lucene-server-%s.jar' % util.VERSION,
+ ('io.netty', 'netty', '3.6.5.Final'),
+ # For Base64 encode/decode:
+ ('commons-codec', 'commons-codec', '1.7'),
+ ('org.apache.lucene', 'lucene-test-framework', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-core', util.LUCENE_VERSION),
+ ('net.minidev', 'json-smart', '1.1.1'),
+ ('org.codehaus.jackson', 'jackson-core-asl', '1.9.12'),
+ ('org.codehaus.jackson', 'jackson-mapper-asl', '1.9.12'),
+ ('junit', 'junit', '4.10'),
+ ('org.apache.lucene', 'lucene-suggest', util.LUCENE_VERSION),
+ ('com.carrotsearch.randomizedtesting', 'junit4-ant', '2.0.9'),
+ ('com.carrotsearch.randomizedtesting', 'randomizedtesting-runner', '2.0.9'),
+ ]
+
+runTestDeps = compileTestDeps + deps + [
+ ('org.apache.lucene', 'lucene-codecs', util.LUCENE_VERSION),
+ ]
+
+def main():
+ b = util.Build('lucene-server', util.VERSION, deps, compileTestDeps,
+ runTestDeps,
+ ('plugins/BinaryDocument',))
+ b.addTestPackage('build/test/MockPlugin-0.1.zip',
+ [('Mock/org/apache/lucene/MockPlugin.class', 'build/classes/test/org/apache/lucene/server/MockPlugin.class'),
+ ('Mock/lucene-server-plugin.properties', 'src/test/org/apache/lucene/server/MockPlugin-lucene-server-plugin.properties'),
+ ('Mock/site/hello.txt', 'src/test/org/apache/lucene/server/MockPlugin-hello.txt')])
+
+ b.run()
+
+if __name__ == '__main__':
+ main()
Added: lucene/dev/branches/lucene5376/lucene/server/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/build.xml?rev=1553272&view=auto
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/build.xml (added)
+++ lucene/dev/branches/lucene5376/lucene/server/build.xml Tue Dec 24 13:39:22 2013
@@ -0,0 +1,95 @@
+<project name="luceneserver" default="jar" basedir="."
+ xmlns:ivy="antlib:org.apache.ivy.ant"
+ xmlns:junit4="antlib:com.carrotsearch.junit4">
+
+ <path id="compile.path">
+ <path refid="dependencies.path"/>
+ </path>
+
+ <path id="test.path">
+ <pathelement location="build/classes/java"/>
+ <path refid="compile.path"/>
+ </path>
+
+ <target name="resolve">
+ <!-- override: just for safety, should be unnecessary -->
+ <ivy:configure file="ivy-settings.xml" override="true"/>
+ <ivy:cachepath pathid="dependencies.path" log="download-only" type="bundle,jar"/>
+ <taskdef resource="com/carrotsearch/junit4/antlib.xml" uri="antlib:com.carrotsearch.junit4" classpathref="dependencies.path"/>
+ </target>
+
+ <target name="clean">
+ <delete dir="build"/>
+ </target>
+
+ <target name="compile" depends="resolve">
+ <compile src="src/java" dest="build/classes/java" classpathref="compile.path"/>
+ </target>
+
+ <target name="compile-test" depends="compile">
+ <compile src="src/test" dest="build/classes/test" classpathref="test.path"/>
+ </target>
+
+ <target name="jar" depends="compile">
+ <jar destfile="build/luceneserver.jar" basedir="build/classes/java"/>
+ </target>
+
+ <macrodef name="compile">
+ <attribute name="src"/>
+ <attribute name="dest"/>
+ <attribute name="classpathref"/>
+ <sequential>
+ <mkdir dir="@{dest}"/>
+ <javac srcdir="@{src}"
+ destdir="@{dest}"
+ classpathref="@{classpathref}"
+ includeantruntime="false"
+ source="1.6"
+ target="1.6"
+ debug="on"
+ encoding="UTF-8">
+ <compilerarg line="-Xlint -Xlint:-serial"/>
+ </javac>
+ </sequential>
+ </macrodef>
+
+ <condition property="tests.class" value="*.${testcase}">
+ <isset property="testcase"/>
+ </condition>
+
+ <condition property="tests.method" value="*.${testmethod}">
+ <isset property="testmethod"/>
+ </condition>
+
+ <property name="tests.jvms" value="auto" />
+
+ <target name="test" depends="compile-test">
+ <sequential>
+ <mkdir dir="build/test"/>
+ <junit4:pickseed property="tests.seed"/>
+ <junit4:junit4 dir="build/test"
+ tempdir="build/test"
+ parallelism="${tests.jvms}">
+ <assertions>
+ <enable/>
+ </assertions>
+ <classpath location="build/classes/test"/>
+ <classpath refid="test.path"/>
+ <fileset dir="build/classes/test">
+ <include name="**/Test*.class"/>
+ <exclude name="**/*$*" />
+ </fileset>
+ <listeners>
+ <junit4:report-text showThrowable="true" showStackTraces="true" showOutput="always" showStatusOk="false"/>
+ </listeners>
+ </junit4:junit4>
+
+ <!-- nocommit is this "right"? -->
+ <subant target="test" inheritall="false" failonerror="true">
+ <fileset dir="plugins/BinaryDocument" includes="build.xml" />
+ </subant>
+
+ </sequential>
+ </target>
+
+</project>
Added: lucene/dev/branches/lucene5376/lucene/server/ivy-settings.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/ivy-settings.xml?rev=1553272&view=auto
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/ivy-settings.xml (added)
+++ lucene/dev/branches/lucene5376/lucene/server/ivy-settings.xml Tue Dec 24 13:39:22 2013
@@ -0,0 +1,51 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<ivysettings>
+ <settings defaultResolver="default"/>
+
+ <property name="local-maven2-dir" value="${user.home}/.m2/repository/" />
+
+ <include url="${ivy.default.settings.dir}/ivysettings-public.xml"/>
+ <include url="${ivy.default.settings.dir}/ivysettings-shared.xml"/>
+ <include url="${ivy.default.settings.dir}/ivysettings-local.xml"/>
+ <include url="${ivy.default.settings.dir}/ivysettings-main-chain.xml"/>
+
+ <!--<caches lockStrategy="artifact-lock"/>-->
+
+ <resolvers>
+ <ibiblio name="apache" root="http://repository.apache.org/content/groups/snapshots" m2compatible="true" />
+
+ <!--
+ <filesystem name="local-maven-2" m2compatible="true" local="true">
+ <artifact
+ pattern="${local-maven2-dir}/[organisation]/[module]/[revision]/[module]-[revision].[ext]" />
+ <ivy
+ pattern="${local-maven2-dir}/[organisation]/[module]/[revision]/[module]-[revision].pom" />
+ </filesystem>
+ -->
+
+ <chain name="default" returnFirst="true" checkmodified="true" changingPattern=".*SNAPSHOT">
+ <resolver ref="local"/>
+ <!-- <resolver ref="local-maven-2" /> -->
+ <resolver ref="main"/>
+ <resolver ref="apache" />
+ </chain>
+ </resolvers>
+
+</ivysettings>
Added: lucene/dev/branches/lucene5376/lucene/server/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/ivy.xml?rev=1553272&view=auto
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/ivy.xml (added)
+++ lucene/dev/branches/lucene5376/lucene/server/ivy.xml Tue Dec 24 13:39:22 2013
@@ -0,0 +1,58 @@
+<!DOCTYPE ivy-module [
+ <!ENTITY lucene.version "4.3-SNAPSHOT">
+]>
+<ivy-module version="2.0">
+ <info organisation="foo" module="lucene-server"/>
+ <dependencies>
+ <dependency org="io.netty" name="netty" rev="3.5.11.Final"/>
+ <!--<dependency org="com.google.code" name="gson" rev="2.2.2"/>-->
+ <dependency org="com.carrotsearch.randomizedtesting" name="junit4-ant" rev="2.0.9"/>
+ <!-- only for Base64 encode/decode I think: -->
+ <dependency org="commons-codec" name="commons-codec" rev="1.7"/>
+
+ <!-- Tika dependencies - see http://tika.apache.org/1.3/gettingstarted.html#Using_Tika_as_a_Maven_dependency -->
+ <!-- When upgrading Tika, upgrade dependencies versions and add any new ones
+ (except slf4j-api, commons-codec, commons-logging, geronimo-stax-api_1.0_spec) -->
+ <!--
+ <dependency org="org.apache.tika" name="tika-core" rev="1.3" transitive="false"/>
+ <dependency org="org.apache.tika" name="tika-parsers" rev="1.3" transitive="false"/>
+ <dependency org="org.gagravarr" name="vorbis-java-tika" rev="0.1" transitive="false"/>
+ <dependency org="org.gagravarr" name="vorbis-java-core" rev="0.1" transitive="false"/>
+ <dependency org="edu.ucar" name="netcdf" rev="4.2-min" transitive="false"/>
+ <dependency org="org.apache.james" name="apache-mime4j-core" rev="0.7.2" transitive="false"/>
+ <dependency org="org.apache.james" name="apache-mime4j-dom" rev="0.7.2" transitive="false"/>
+ <dependency org="org.apache.commons" name="commons-compress" rev="1.4.1" transitive="false"/>
+ <dependency org="org.apache.pdfbox" name="pdfbox" rev="1.7.1" transitive="false"/>
+ <dependency org="org.apache.pdfbox" name="fontbox" rev="1.7.1" transitive="false"/>
+ <dependency org="org.apache.pdfbox" name="jempbox" rev="1.7.1" transitive="false"/>
+ <dependency org="org.bouncycastle" name="bcmail-jdk15" rev="1.45" transitive="false"/>
+ <dependency org="org.bouncycastle" name="bcprov-jdk15" rev="1.45" transitive="false"/>
+ <dependency org="org.apache.poi" name="poi" rev="3.8" transitive="false"/>
+ <dependency org="org.apache.poi" name="poi-scratchpad" rev="3.8" transitive="false"/>
+ <dependency org="org.apache.poi" name="poi-ooxml" rev="3.8" transitive="false"/>
+ <dependency org="org.apache.poi" name="poi-ooxml-schemas" rev="3.8" transitive="false"/>
+ <dependency org="org.apache.xmlbeans" name="xmlbeans" rev="2.3.0" transitive="false"/>
+ <dependency org="dom4j" name="dom4j" rev="1.6.1" transitive="false"/>
+ <dependency org="org.ccil.cowan.tagsoup" name="tagsoup" rev="1.2.1" transitive="false"/>
+ <dependency org="com.googlecode.mp4parser" name="isoparser" rev="1.0-RC-1" transitive="false"/>
+ <dependency org="com.drewnoakes" name="metadata-extractor" rev="2.6.2" transitive="false"/>
+ <dependency org="de.l3s.boilerpipe" name="boilerpipe" rev="1.1.0" transitive="false"/>
+ <dependency org="rome" name="rome" rev="0.9" transitive="false"/>
+ <dependency org="jdom" name="jdom" rev="1.0" transitive="false"/>
+ <dependency org="com.googlecode.juniversalchardet" name="juniversalchardet" rev="1.0.3" transitive="false"/>
+ <dependency org="org.tukaani" name="xz" rev="1.0" transitive="false"/>
+ -->
+
+ <dependency org="org.apache.lucene" name="lucene-test-framework" rev="&lucene.version;"/>
+ <dependency org="org.apache.lucene" name="lucene-queryparser" rev="&lucene.version;"/>
+ <dependency org="org.apache.lucene" name="lucene-misc" rev="&lucene.version;"/>
+ <dependency org="org.apache.lucene" name="lucene-highlighter" rev="&lucene.version;"/>
+ <dependency org="org.apache.lucene" name="lucene-suggest" rev="&lucene.version;"/>
+ <dependency org="org.apache.lucene" name="lucene-grouping" rev="&lucene.version;"/>
+ <dependency org="org.apache.lucene" name="lucene-analyzers-common" rev="&lucene.version;"/>
+ <dependency org="org.apache.lucene" name="lucene-facet" rev="&lucene.version;"/>
+ <dependency org="net.minidev" name="json-smart" rev="1.1.1"/>
+ <dependency org="org.codehaus.jackson" name="jackson-core-asl" rev="1.9.12"/>
+ <dependency org="org.codehaus.jackson" name="jackson-mapper-asl" rev="1.9.12"/>
+ </dependencies>
+</ivy-module>
Added: lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/build.py
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/build.py?rev=1553272&view=auto
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/build.py (added)
+++ lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/build.py Tue Dec 24 13:39:22 2013
@@ -0,0 +1,112 @@
+import sys
+import os
+import shutil
+
+VERSION = '0.1-SNAPSHOT'
+
+sys.path.insert(0, '../../src/python')
+import util
+
+SERVER_JAR = '../../build/lucene-server-%s.jar' % util.VERSION
+
+deps = [
+ ('org.apache.tika', 'tika-app', '1.3'),
+ ('net.minidev', 'json-smart', '1.1.1'),
+ ('org.codehaus.jackson', 'jackson-core-asl', '1.9.12'),
+ ('org.codehaus.jackson', 'jackson-mapper-asl', '1.9.12'),
+ SERVER_JAR,
+ ]
+
+compileTestDeps = [
+ ('junit', 'junit', '4.10'),
+ ('com.carrotsearch.randomizedtesting', 'junit4-ant', '2.0.9'),
+ ('com.carrotsearch.randomizedtesting', 'randomizedtesting-runner', '2.0.9'),
+ ('org.apache.lucene', 'lucene-codecs', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-test-framework', util.LUCENE_VERSION),
+ '../../build/classes/test/',
+ ]
+
+compileTestDeps.extend((
+ ('io.netty', 'netty', '3.6.5.Final'),
+ # For Base64 encode/decode:
+ ('commons-codec', 'commons-codec', '1.7'),
+ ('org.apache.lucene', 'lucene-queryparser', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-misc', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-queries', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-highlighter', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-suggest', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-grouping', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-analyzers-common', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-facet', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-core', util.LUCENE_VERSION),
+ ('net.minidev', 'json-smart', '1.1.1'),
+ ('org.codehaus.jackson', 'jackson-core-asl', '1.9.12'),
+ ('org.codehaus.jackson', 'jackson-mapper-asl', '1.9.12'),
+ ))
+
+compileTestDeps.append(SERVER_JAR)
+
+coreDeps = [
+ ('io.netty', 'netty', '3.6.5.Final'),
+ # For Base64 encode/decode:
+ ('commons-codec', 'commons-codec', '1.7'),
+ ('org.apache.lucene', 'lucene-queryparser', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-misc', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-queries', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-highlighter', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-suggest', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-grouping', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-analyzers-common', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-analyzers-icu', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-facet', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-core', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-join', util.LUCENE_VERSION),
+ ('com.ibm.icu', 'icu4j', '49.1'),
+ ('net.minidev', 'json-smart', '1.1.1'),
+ ('org.codehaus.jackson', 'jackson-core-asl', '1.9.12'),
+ ('org.codehaus.jackson', 'jackson-mapper-asl', '1.9.12'),
+ ]
+
+coreCompileTestDeps = [
+ ('io.netty', 'netty', '3.6.5.Final'),
+ # For Base64 encode/decode:
+ ('commons-codec', 'commons-codec', '1.7'),
+ ('org.apache.lucene', 'lucene-test-framework', util.LUCENE_VERSION),
+ ('org.apache.lucene', 'lucene-core', util.LUCENE_VERSION),
+ ('net.minidev', 'json-smart', '1.1.1'),
+ ('org.codehaus.jackson', 'jackson-core-asl', '1.9.12'),
+ ('org.codehaus.jackson', 'jackson-mapper-asl', '1.9.12'),
+ ('junit', 'junit', '4.10'),
+ ('org.apache.lucene', 'lucene-suggest', util.LUCENE_VERSION),
+ ('com.carrotsearch.randomizedtesting', 'junit4-ant', '2.0.9'),
+ ('com.carrotsearch.randomizedtesting', 'randomizedtesting-runner', '2.0.9'),
+ ]
+
+runTestDeps = coreCompileTestDeps + coreDeps + [
+ ('org.apache.lucene', 'lucene-codecs', util.LUCENE_VERSION),
+ SERVER_JAR,
+ '../../build/classes/test',
+ ]
+
+def main():
+
+ b = util.Build('BinaryDocument', VERSION, deps, compileTestDeps, runTestDeps,
+ packageFiles=(
+ ('BinaryDocument/BinaryDocument-%s.jar' % VERSION, 'build/BinaryDocument-%s.jar' % VERSION),
+ ('BinaryDocument/lib/tika-app-1.3.jar', util.getJAR(util.LUCENE_ROOT, 'org.apache.tika', 'tika-app', '1.3')),
+ ))
+
+ # We always package if running tests because tests install the
+ # plugin from the zip file:
+ if b.doTest:
+ b.doPackage = True
+
+ if not b.coreDone:
+ util.pushDir('../..')
+ util.run('python -u build.py compile-test')
+ util.popDir()
+
+ b.run()
+
+if __name__ == '__main__':
+ main()
Added: lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/build.xml?rev=1553272&view=auto
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/build.xml (added)
+++ lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/build.xml Tue Dec 24 13:39:22 2013
@@ -0,0 +1,102 @@
+<project name="BinaryDocument" default="jar" basedir="."
+ xmlns:ivy="antlib:org.apache.ivy.ant"
+ xmlns:junit4="antlib:com.carrotsearch.junit4">
+
+ <path id="compile.path">
+ <path refid="dependencies.path"/>
+ <pathelement location="../../build/classes/java"/>
+ </path>
+
+ <path id="test.compile.path">
+ <pathelement location="build/classes/java"/>
+ <pathelement location="../../build/classes/test"/>
+ <path refid="compile.path"/>
+ </path>
+
+ <path id="test.path">
+ <pathelement location="../../build/classes/test"/>
+ <path refid="compile.path"/>
+ </path>
+
+ <target name="resolve">
+ <!-- override: just for safety, should be unnecessary -->
+ <ivy:configure file="ivy-settings.xml" override="true"/>
+ <ivy:cachepath pathid="dependencies.path" log="download-only" type="bundle,jar"/>
+ <taskdef resource="com/carrotsearch/junit4/antlib.xml" uri="antlib:com.carrotsearch.junit4" classpathref="dependencies.path"/>
+ </target>
+
+ <target name="clean">
+ <delete dir="build"/>
+ </target>
+
+ <target name="compile" depends="resolve">
+ <compile src="src/java" dest="build/classes/java" classpathref="compile.path"/>
+ <copy todir="@{dest}">
+ <fileset dir="src/resources"/>
+ </copy>
+ </target>
+
+ <target name="compile-test" depends="compile">
+ <compile src="src/test" dest="build/classes/test" classpathref="test.compile.path"/>
+ </target>
+
+ <target name="jar" depends="compile">
+ <jar destfile="build/BinaryDocumentPlugin.jar">
+ <fileset dir="build/classes/java"/>
+ <fileset dir="src/resources"/>
+ </jar>
+ </target>
+
+ <macrodef name="compile">
+ <attribute name="src"/>
+ <attribute name="dest"/>
+ <attribute name="classpathref"/>
+ <sequential>
+ <mkdir dir="@{dest}"/>
+ <javac srcdir="@{src}"
+ destdir="@{dest}"
+ classpathref="@{classpathref}"
+ includeantruntime="false"
+ source="1.6"
+ target="1.6"
+ debug="on"
+ encoding="UTF-8">
+ <compilerarg line="-Xlint -Xlint:-serial"/>
+ </javac>
+ </sequential>
+ </macrodef>
+
+ <condition property="tests.class" value="*.${testcase}">
+ <isset property="testcase"/>
+ </condition>
+
+ <condition property="tests.method" value="*.${testmethod}">
+ <isset property="testmethod"/>
+ </condition>
+
+ <property name="tests.jvms" value="auto" />
+
+ <target name="test" depends="compile-test,jar">
+ <sequential>
+ <mkdir dir="build/test"/>
+ <junit4:pickseed property="tests.seed"/>
+ <junit4:junit4 dir="build/test"
+ tempdir="build/test"
+ parallelism="${tests.jvms}">
+ <assertions>
+ <enable/>
+ </assertions>
+ <classpath location="build/classes/test"/>
+ <classpath refid="test.path"/>
+ <fileset dir="build/classes/test">
+ <include name="**/Test*.class"/>
+ <exclude name="**/*$*" />
+ </fileset>
+ <listeners>
+ <junit4:report-text showThrowable="true" showStackTraces="true" showOutput="always" showStatusOk="false"/>
+ </listeners>
+ </junit4:junit4>
+ </sequential>
+ </target>
+
+</project>
Added: lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/dist.py
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/dist.py?rev=1553272&view=auto
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/dist.py (added)
+++ lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/dist.py Tue Dec 24 13:39:22 2013
@@ -0,0 +1,27 @@
+import sys
+import re
+import zipfile
+import os
+import shutil
+
+sys.path.insert(0, '../../src/python')
+import util
+
+util.run('ant jar')
+
+if os.path.exists('dist'):
+ shutil.rmtree('dist')
+
+os.makedirs('dist')
+
+reComment = re.compile('<!--.*?-->', re.DOTALL)
+reDep = re.compile('<dependency(.*?)/>')
+
+jars = []
+s = open('ivy.xml', 'rb').read()
+s = reComment.sub('', s)
+for m in reDep.findall(s):
+ print 'HERE: %s' % m
+
+with zipfile.ZipFile('dist/BinaryDocument.zip', 'w') as zf:
+ zf.write('build/BinaryDocumentPlugin.jar', 'BinaryDocument/BinaryDocumentPlugin.jar')
Added: lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/ivy-settings.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/ivy-settings.xml?rev=1553272&view=auto
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/ivy-settings.xml (added)
+++ lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/ivy-settings.xml Tue Dec 24 13:39:22 2013
@@ -0,0 +1,51 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<ivysettings>
+ <settings defaultResolver="default"/>
+
+ <property name="local-maven2-dir" value="${user.home}/.m2/repository/" />
+
+ <include url="${ivy.default.settings.dir}/ivysettings-public.xml"/>
+ <include url="${ivy.default.settings.dir}/ivysettings-shared.xml"/>
+ <include url="${ivy.default.settings.dir}/ivysettings-local.xml"/>
+ <include url="${ivy.default.settings.dir}/ivysettings-main-chain.xml"/>
+
+ <!--<caches lockStrategy="artifact-lock"/>-->
+
+ <resolvers>
+ <ibiblio name="apache" root="http://repository.apache.org/content/groups/snapshots" m2compatible="true" />
+
+ <!--
+ <filesystem name="local-maven-2" m2compatible="true" local="true">
+ <artifact
+ pattern="${local-maven2-dir}/[organisation]/[module]/[revision]/[module]-[revision].[ext]" />
+ <ivy
+ pattern="${local-maven2-dir}/[organisation]/[module]/[revision]/[module]-[revision].pom" />
+ </filesystem>
+ -->
+
+ <chain name="default" returnFirst="true" checkmodified="true" changingPattern=".*SNAPSHOT">
+ <resolver ref="local"/>
+ <!-- <resolver ref="local-maven-2" /> -->
+ <resolver ref="main"/>
+ <resolver ref="apache" />
+ </chain>
+ </resolvers>
+
+</ivysettings>
Added: lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/ivy.xml?rev=1553272&view=auto
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/ivy.xml (added)
+++ lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/ivy.xml Tue Dec 24 13:39:22 2013
@@ -0,0 +1,60 @@
+<!DOCTYPE ivy-module [
+ <!ENTITY lucene.version "4.3-SNAPSHOT">
+]>
+<ivy-module version="2.0">
+ <info organisation="foo" module="lucene-server-BinaryDocument"/>
+ <dependencies>
+ <dependency org="io.netty" name="netty" rev="3.5.11.Final"/>
+ <dependency org="com.carrotsearch.randomizedtesting" name="junit4-ant" rev="2.0.9"/>
+ <!-- only for Base64 encode/decode I think: -->
+ <dependency org="commons-codec" name="commons-codec" rev="1.7"/>
+
+ <!-- Tika dependencies - see http://tika.apache.org/1.3/gettingstarted.html#Using_Tika_as_a_Maven_dependency -->
+ <!-- When upgrading Tika, upgrade dependencies versions and add any new ones
+ (except slf4j-api, commons-codec, commons-logging, geronimo-stax-api_1.0_spec) -->
+
+ <dependency org="org.apache.tika" name="tika-app" rev="1.3" transitive="false"/>
+
+ <!--
+ <dependency org="org.apache.tika" name="tika-core" rev="1.3" transitive="false"/>
+ <dependency org="org.apache.tika" name="tika-parsers" rev="1.3" transitive="false"/>
+ <dependency org="org.gagravarr" name="vorbis-java-tika" rev="0.1" transitive="false"/>
+ <dependency org="org.gagravarr" name="vorbis-java-core" rev="0.1" transitive="false"/>
+ <dependency org="edu.ucar" name="netcdf" rev="4.2-min" transitive="false"/>
+ <dependency org="org.apache.james" name="apache-mime4j-core" rev="0.7.2" transitive="false"/>
+ <dependency org="org.apache.james" name="apache-mime4j-dom" rev="0.7.2" transitive="false"/>
+ <dependency org="org.apache.commons" name="commons-compress" rev="1.4.1" transitive="false"/>
+ <dependency org="org.apache.pdfbox" name="pdfbox" rev="1.7.1" transitive="false"/>
+ <dependency org="org.apache.pdfbox" name="fontbox" rev="1.7.1" transitive="false"/>
+ <dependency org="org.apache.pdfbox" name="jempbox" rev="1.7.1" transitive="false"/>
+ <dependency org="org.bouncycastle" name="bcmail-jdk15" rev="1.45" transitive="false"/>
+ <dependency org="org.bouncycastle" name="bcprov-jdk15" rev="1.45" transitive="false"/>
+ <dependency org="org.apache.poi" name="poi" rev="3.8" transitive="false"/>
+ <dependency org="org.apache.poi" name="poi-scratchpad" rev="3.8" transitive="false"/>
+ <dependency org="org.apache.poi" name="poi-ooxml" rev="3.8" transitive="false"/>
+ <dependency org="org.apache.poi" name="poi-ooxml-schemas" rev="3.8" transitive="false"/>
+ <dependency org="org.apache.xmlbeans" name="xmlbeans" rev="2.3.0" transitive="false"/>
+ <dependency org="dom4j" name="dom4j" rev="1.6.1" transitive="false"/>
+ <dependency org="org.ccil.cowan.tagsoup" name="tagsoup" rev="1.2.1" transitive="false"/>
+ <dependency org="com.googlecode.mp4parser" name="isoparser" rev="1.0-RC-1" transitive="false"/>
+ <dependency org="com.drewnoakes" name="metadata-extractor" rev="2.6.2" transitive="false"/>
+ <dependency org="de.l3s.boilerpipe" name="boilerpipe" rev="1.1.0" transitive="false"/>
+ <dependency org="rome" name="rome" rev="0.9" transitive="false"/>
+ <dependency org="jdom" name="jdom" rev="1.0" transitive="false"/>
+ <dependency org="com.googlecode.juniversalchardet" name="juniversalchardet" rev="1.0.3" transitive="false"/>
+ <dependency org="org.tukaani" name="xz" rev="1.0" transitive="false"/>
+ -->
+
+ <dependency org="org.apache.lucene" name="lucene-test-framework" rev="&lucene.version;"/>
+ <dependency org="org.apache.lucene" name="lucene-queryparser" rev="&lucene.version;"/>
+ <dependency org="org.apache.lucene" name="lucene-misc" rev="&lucene.version;"/>
+ <dependency org="org.apache.lucene" name="lucene-highlighter" rev="&lucene.version;"/>
+ <dependency org="org.apache.lucene" name="lucene-suggest" rev="&lucene.version;"/>
+ <dependency org="org.apache.lucene" name="lucene-grouping" rev="&lucene.version;"/>
+ <dependency org="org.apache.lucene" name="lucene-analyzers-common" rev="&lucene.version;"/>
+ <dependency org="org.apache.lucene" name="lucene-facet" rev="&lucene.version;"/>
+ <dependency org="net.minidev" name="json-smart" rev="1.1.1"/>
+ <dependency org="org.codehaus.jackson" name="jackson-core-asl" rev="1.9.12"/>
+ <dependency org="org.codehaus.jackson" name="jackson-mapper-asl" rev="1.9.12"/>
+ </dependencies>
+</ivy-module>
Added: lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/java/org/apache/lucene/server/handlers/CrackDocumentHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/java/org/apache/lucene/server/handlers/CrackDocumentHandler.java?rev=1553272&view=auto
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/java/org/apache/lucene/server/handlers/CrackDocumentHandler.java (added)
+++ lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/java/org/apache/lucene/server/handlers/CrackDocumentHandler.java Tue Dec 24 13:39:22 2013
@@ -0,0 +1,117 @@
+package org.apache.lucene.server.handlers;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.lucene.server.FinishRequest;
+import org.apache.lucene.server.GlobalState;
+import org.apache.lucene.server.IndexState;
+import org.apache.lucene.server.handlers.Handler;
+import org.apache.lucene.server.params.IntType;
+import org.apache.lucene.server.params.Param;
+import org.apache.lucene.server.params.Request;
+import org.apache.lucene.server.params.StringType;
+import org.apache.lucene.server.params.StructType;
+import org.apache.lucene.server.params.Type;
+import org.apache.lucene.server.plugins.BinaryDocumentPlugin;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.parser.PasswordProvider;
+import org.apache.tika.sax.BodyContentHandler;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.SAXException;
+import net.minidev.json.JSONArray;
+import net.minidev.json.JSONObject;
+
+public class CrackDocumentHandler extends Handler {
+ private final static StructType TYPE =
+ new StructType(new Param("indexName", "Index name", new StringType()),
+ new Param("content", "Base64 encoded binary content", new StringType()),
+ new Param("contentType", "Mime type, if known (for example, application/pdf); otherwise the mime type will be autodetected", new StringType()),
+ new Param("fileName", "File name, if known; otherwise the mime type will be autodetected", new StringType()),
+ new Param("password", "Password, if necessary", new StringType()),
+ new Param("maxLength", "Maximum length in bytes of extracted text, or -1 to disable the limit", new IntType(), 1024*1024));
+
+ @Override
+ public StructType getType() {
+ return TYPE;
+ }
+
+ @Override
+ public String getTopDoc() {
+ return "Parses the provided binary document and extracts and returns all available text and metadata; this does not alter the index, and is useful for diagnostic purposes.";
+ }
+
+ public CrackDocumentHandler(GlobalState state) {
+ super(state);
+ }
+
+ @Override
+ public FinishRequest handle(final IndexState state, Request r, Map<String,List<String>> params) throws Exception {
+ byte[] decodedBytes = Base64.decodeBase64(r.getString("content"));
+ ParseContext parseContext = new ParseContext();
+ parseContext.set(Parser.class, BinaryDocumentPlugin.tikaAutoDetectParser);
+ Metadata md = new Metadata();
+ if (r.hasParam("fileName")) {
+ md.add(Metadata.RESOURCE_NAME_KEY, r.getString("fileName"));
+ }
+ if (r.hasParam("contentType")) {
+ md.add(Metadata.CONTENT_TYPE, r.getString("contentType"));
+ }
+ if (r.hasParam("password")) {
+ final String password = r.getString("password");
+ parseContext.set(PasswordProvider.class, new PasswordProvider() {
+ @Override
+ public String getPassword(Metadata metadata) {
+ return password;
+ }
+ });
+ }
+
+ final JSONObject result = new JSONObject();
+
+ ContentHandler handler = new BodyContentHandler(r.getInt("maxLength"));
+ BinaryDocumentPlugin.tikaAutoDetectParser.parse(TikaInputStream.get(decodedBytes),
+ handler, md, parseContext);
+ result.put("body", handler.toString());
+ for(String name : md.names()) {
+ if (md.isMultiValued(name)) {
+ JSONArray arr = new JSONArray();
+ result.put(name, arr);
+ for(String value : md.getValues(name)) {
+ arr.add(value);
+ }
+ } else {
+ result.put(name, md.get(name));
+ }
+ }
+
+ return new FinishRequest() {
+ @Override
+ public String finish() throws IOException {
+ return result.toString();
+ }
+ };
+ }
+}
Added: lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/java/org/apache/lucene/server/plugins/BinaryDocumentPlugin.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/java/org/apache/lucene/server/plugins/BinaryDocumentPlugin.java?rev=1553272&view=auto
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/java/org/apache/lucene/server/plugins/BinaryDocumentPlugin.java (added)
+++ lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/java/org/apache/lucene/server/plugins/BinaryDocumentPlugin.java Tue Dec 24 13:39:22 2013
@@ -0,0 +1,278 @@
+package org.apache.lucene.server.plugins;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.lucene.server.FieldDef;
+import org.apache.lucene.server.GlobalState;
+import org.apache.lucene.server.IndexState.DocumentAndFacets;
+import org.apache.lucene.server.IndexState;
+import org.apache.lucene.server.handlers.AddDocumentHandler;
+import org.apache.lucene.server.handlers.BulkUpdateDocumentHandler;
+import org.apache.lucene.server.handlers.CrackDocumentHandler;
+import org.apache.lucene.server.params.IntType;
+import org.apache.lucene.server.params.Param;
+import org.apache.lucene.server.params.Request;
+import org.apache.lucene.server.params.StringType;
+import org.apache.lucene.server.params.StructType;
+import org.apache.lucene.server.plugins.Plugin;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.AutoDetectParser;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.parser.PasswordProvider;
+import org.apache.tika.sax.BodyContentHandler;
+import org.codehaus.jackson.JsonParser;
+import org.codehaus.jackson.JsonToken;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.SAXException;
+
+// TODO
+// - allow pre-registering the mappings in settings/fields
+
+public class BinaryDocumentPlugin extends Plugin {
+
+ @Override
+ public String getTopDoc() {
+ return "Parses non-plain-text document formats, such as Adobe PDF, Microsoft Office, HTML, into separate fields with plain-text values, using Apache Tika. This plugin modifies the indexing methods to accept an optional per-document binary value, and adds a new @crackDocument method.";
+ }
+
+ @Override
+ public String getName() {
+ return "BinaryDocument";
+ }
+
+ public static final Parser tikaAutoDetectParser = new AutoDetectParser();
+
+ private static class AddDocumentPostHandler implements AddDocumentHandler.PostHandle {
+
+ @Override
+ public void invoke(IndexState state, Request r, DocumentAndFacets doc) throws IOException {
+ if (r.hasParam("binary")) {
+ // Parse binary document using Tika:
+ Request r2 = r.getStruct("binary");
+ byte[] decodedBytes = Base64.decodeBase64(r2.getString("content"));
+ ParseContext parseContext = new ParseContext();
+ parseContext.set(Parser.class, tikaAutoDetectParser);
+ Metadata md = new Metadata();
+ if (r2.hasParam("fileName")) {
+ md.add(Metadata.RESOURCE_NAME_KEY, r2.getString("fileName"));
+ }
+ if (r2.hasParam("contentType")) {
+ md.add(Metadata.CONTENT_TYPE, r2.getString("contentType"));
+ }
+ if (r2.hasParam("password")) {
+ final String password = r2.getString("password");
+ parseContext.set(PasswordProvider.class, new PasswordProvider() {
+ @Override
+ public String getPassword(Metadata metadata) {
+ return password;
+ }
+ });
+ }
+
+ // nocommit this discards non-body text!
+ ContentHandler handler = new BodyContentHandler(r2.getInt("maxLength"));
+ try {
+ tikaAutoDetectParser.parse(TikaInputStream.get(decodedBytes),
+ handler, md, parseContext);
+ } catch (SAXException se) {
+ throw new IllegalArgumentException("exception while parsing document", se);
+ } catch (TikaException te) {
+ throw new IllegalArgumentException("exception while parsing document", te);
+ }
+
+ Request mappings = r2.getStruct("mappings");
+ Iterator<Map.Entry<String,Object>> it = mappings.getParams();
+ while (it.hasNext()) {
+ Map.Entry<String,Object> ent = it.next();
+ if (!(ent.getValue() instanceof String)) {
+ r2.fail(ent.getKey(), "value must be a string; got " + ent.getValue().getClass());
+ }
+ String fieldName = (String) ent.getValue();
+ String value;
+ if (ent.getKey().equals("body")) {
+ value = handler.toString();
+ } else {
+ // nocommit what about multi-valued?
+ value = md.get(ent.getKey());
+ }
+ if (value != null) {
+ FieldDef fd = state.getField(fieldName);
+ // TODO: how to pass boost?
+ AddDocumentHandler.parseOneValue(fd, doc, AddDocumentHandler.fixType(fd, value), 1.0f);
+ }
+ }
+ r2.clearParams();
+ }
+ }
+
+ static void parseBinary(IndexState state, DocumentAndFacets doc, JsonParser p) throws IOException {
+ JsonToken token = p.nextToken();
+ if (token != JsonToken.START_OBJECT) {
+ throw new IllegalArgumentException("binary should be an object");
+ }
+ Metadata md = new Metadata();
+ Map<String,String> mappings = new HashMap<String,String>();
+ ParseContext parseContext = new ParseContext();
+ byte[] decodedBytes = null;
+ int maxLength = -1;
+ while (true) {
+ token = p.nextToken();
+ if (token == JsonToken.END_OBJECT) {
+ break;
+ }
+ assert token == JsonToken.FIELD_NAME: token;
+ String fieldName = p.getText();
+ if (fieldName.equals("content")) {
+ token = p.nextToken();
+ if (token != JsonToken.VALUE_STRING) {
+ throw new IllegalArgumentException("binary > contents should be a Base64 encoded string");
+ }
+ decodedBytes = Base64.decodeBase64(p.getText());
+ } else if (fieldName.equals("fileName")) {
+ token = p.nextToken();
+ if (token != JsonToken.VALUE_STRING) {
+ throw new IllegalArgumentException("binary > fileName should be a string");
+ }
+ md.add(Metadata.RESOURCE_NAME_KEY, p.getText());
+ } else if (fieldName.equals("contentType")) {
+ token = p.nextToken();
+ if (token != JsonToken.VALUE_STRING) {
+ throw new IllegalArgumentException("binary > contentType should be a string");
+ }
+ md.add(Metadata.CONTENT_TYPE, p.getText());
+ } else if (fieldName.equals("password")) {
+ token = p.nextToken();
+ if (token != JsonToken.VALUE_STRING) {
+ throw new IllegalArgumentException("binary > password should be a string");
+ }
+ final String password = p.getText();
+ parseContext.set(PasswordProvider.class, new PasswordProvider() {
+ @Override
+ public String getPassword(Metadata metadata) {
+ return password;
+ }
+ });
+ } else if (fieldName.equals("maxLength")) {
+ if (token != JsonToken.VALUE_NUMBER_INT) {
+ throw new IllegalArgumentException("binary > maxLength should be an int");
+ }
+ maxLength = p.getIntValue();
+ } else if (fieldName.equals("mappings")) {
+ token = p.nextToken();
+ if (token != JsonToken.START_OBJECT) {
+ throw new IllegalArgumentException("binary > mappings should be an object");
+ }
+ while (true) {
+ token = p.nextToken();
+ if (token == JsonToken.END_OBJECT) {
+ break;
+ }
+ assert token == JsonToken.FIELD_NAME;
+ fieldName = p.getText();
+ token = p.nextToken();
+ if (token != JsonToken.VALUE_STRING) {
+ throw new IllegalArgumentException("binary > mappings > " + fieldName + " should be a string");
+ }
+ mappings.put(fieldName, p.getText());
+ }
+ } else {
+ throw new IllegalArgumentException("binary: unrecognized argument " + fieldName);
+ }
+ }
+
+ if (decodedBytes == null) {
+ throw new IllegalArgumentException("binary > content is missing");
+ }
+
+ ContentHandler handler = new BodyContentHandler(maxLength);
+ try {
+ tikaAutoDetectParser.parse(TikaInputStream.get(decodedBytes),
+ handler, md, parseContext);
+ } catch (SAXException se) {
+ throw new IllegalArgumentException("exception while parsing document", se);
+ } catch (TikaException te) {
+ throw new IllegalArgumentException("exception while parsing document", te);
+ }
+
+ // Used only for errors:
+ Request r = new Request(null, "binary", null, null);
+
+ for(Map.Entry<String,String> ent : mappings.entrySet()) {
+ String fieldName = ent.getValue();
+ String value;
+ if (ent.getKey().equals("body")) {
+ value = handler.toString();
+ } else {
+ // nocommit what about multi-valued?
+ value = md.get(ent.getKey());
+ }
+ if (value != null) {
+ FieldDef fd = state.getField(fieldName);
+ AddDocumentHandler.parseOneValue(fd, doc, AddDocumentHandler.fixType(fd, value), 1.0f);
+ }
+ }
+ }
+
+ @Override
+ public boolean invoke(IndexState state, String fieldName, JsonParser p, DocumentAndFacets doc) throws IOException {
+ if (fieldName.equals("binary")) {
+ parseBinary(state, doc, p);
+ return true;
+ } else {
+ return false;
+ }
+ }
+ }
+
+ public BinaryDocumentPlugin(GlobalState state) {
+
+ state.addHandler("crackDocument", new CrackDocumentHandler(state));
+
+ // Register our pre-processor in addDocument:
+ AddDocumentHandler addDocHandler = (AddDocumentHandler) state.getHandler("addDocument");
+ addDocHandler.addPostHandle(new AddDocumentPostHandler());
+
+ Param binParam = new Param("binary", "Binary document to extract text and metadata fields from. To see all metadata available for a given document without indexing it, use @crackDocument.",
+ new StructType(
+ new Param("content", "Base64 encoded binary content", new StringType()),
+ new Param("contentType", "Mime type, if known (for example, application/pdf); otherwise the mime type will be autodetected", new StringType()),
+ new Param("fileName", "File name, if known; otherwise the mime type will be autodetected", new StringType()),
+ new Param("password", "Password, if necessary", new StringType()),
+ new Param("maxLength", "Maximum length in bytes of extracted text, or -1 to disable the limit", new IntType(), 1024*1024),
+ new Param("mappings", "How to map the document body and metadata to registered fields; this is an object whose keys are 'body' or Dublin Core metadata keys, and whose values are the registered field names to map to.", new StructType())));
+
+ // nocommit messy: this is a static ... if we are
+ // installed more than once then we double-add!
+ AddDocumentHandler.DOCUMENT_TYPE.addParam(binParam);
+ state.getHandler("addDocument").getType().addParam(binParam);
+ state.getHandler("updateDocument").getType().addParam(binParam);
+
+ // nocommit messy: this is a static ... if we are
+ // installed more than once then we double-add!
+ BulkUpdateDocumentHandler.UPDATE_DOCUMENT_TYPE.addParam(binParam);
+ }
+}
Added: lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/resources/lucene-server-plugin.properties
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/resources/lucene-server-plugin.properties?rev=1553272&view=auto
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/resources/lucene-server-plugin.properties (added)
+++ lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/resources/lucene-server-plugin.properties Tue Dec 24 13:39:22 2013
@@ -0,0 +1 @@
+class: org.apache.lucene.server.plugins.BinaryDocumentPlugin
Added: lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/test-resources/test.docx
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/test-resources/test.docx?rev=1553272&view=auto
==============================================================================
Binary file - no diff available.
Added: lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/test/org/apache/lucene/server/TestBinaryDocuments.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/test/org/apache/lucene/server/TestBinaryDocuments.java?rev=1553272&view=auto
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/test/org/apache/lucene/server/TestBinaryDocuments.java (added)
+++ lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/src/test/org/apache/lucene/server/TestBinaryDocuments.java Tue Dec 24 13:39:22 2013
@@ -0,0 +1,153 @@
+package org.apache.lucene.server;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.util.Locale;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.lucene.util._TestUtil;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import net.minidev.json.JSONArray;
+import net.minidev.json.JSONObject;
+
+public class TestBinaryDocuments extends ServerBaseTestCase {
+
+ @BeforeClass
+ public static void initClass() throws Exception {
+ clearDir();
+
+ // Install the BinaryDocumentPlugin:
+ File zipFile = new File("../../../dist/BinaryDocument-0.1-SNAPSHOT.zip");
+ if (!zipFile.exists()) {
+ throw new RuntimeException(zipFile.getCanonicalPath() + " does not exist");
+ }
+
+ installPlugin(zipFile);
+ // nocommit use ant / properties / something cleaner...:
+ /*
+ File destDir = new File("state/plugins/BinaryDocument/");
+ destDir.mkdirs();
+ File srcFile = new File("../../BinaryDocument-0.1.jar");
+ if (!srcFile.exists()) {
+ // nocommit get build.xml to do this!
+ throw new RuntimeException("run ant jar first: " + srcFile.getCanonicalPath() + " does not exist");
+ }
+ File destFile = new File(destDir, "BinaryDocumentPlugin.jar");
+ copyFile(srcFile, destFile);
+ */
+
+ startServer();
+ createAndStartIndex();
+ registerFields();
+ commit();
+ }
+
+ @AfterClass
+ public static void fini() throws Exception {
+ shutdownServer();
+ System.clearProperty("sun.nio.ch.bugLevel"); // hack WTF
+ }
+
+ private static void registerFields() throws Exception {
+ JSONObject o = new JSONObject();
+ put(o, "body", "{type: text, highlight: true, store: true, analyzer: {class: StandardAnalyzer, matchVersion: LUCENE_43}, similarity: {class: BM25Similarity, b: 0.15}}");
+ put(o, "author", "{type: text, index: false, facet: flat, store: true, group: true}");
+ put(o, "charCount", "{type: int, store: true}");
+ JSONObject o2 = new JSONObject();
+ o2.put("indexName", "index");
+ o2.put("fields", o);
+ send("registerFields", o2);
+ }
+
+ public void testCrackDocument() throws Exception {
+ // nocommit get test-resources onto classpath so I don't have to
+ // hardwire full path!
+ RandomAccessFile f = new RandomAccessFile("../../../src/test-resources/test.docx", "r");
+ byte[] b = new byte[(int)f.length()];
+ f.read(b);
+
+ // Make sure we have docs for crackDocument:
+ assertTrue(httpLoad("doc?method=crackDocument").indexOf("<h1>Lucene Server: crackDocument</h1>") != -1);
+
+ String enc = Base64.encodeBase64String(b);
+ JSONObject result = send("crackDocument", "{indexName: index, content: '" + enc + "', fileName: 'test.docx'}");
+ assertEquals("Word document with a wee bit of text in it.\n", result.get("body"));
+ assertEquals("Michael McCandless", result.get("Author"));
+ }
+
+ public void testWordDoc() throws Exception {
+ send("deleteAllDocuments", "{indexName: index}");
+
+ // nocommit get test-resources onto classpath so I don't have to
+ // hardwire full path!
+ RandomAccessFile f = new RandomAccessFile("../../../src/test-resources/test.docx", "r");
+ byte[] b = new byte[(int)f.length()];
+ f.read(b);
+
+ // Make sure doc is updated w/ new binary param:
+ assertTrue(httpLoad("doc?method=addDocument").indexOf("<b>binary</b>") != -1);
+ assertTrue(httpLoad("doc?method=addDocuments").indexOf("<b>binary</b>") != -1);
+ assertTrue(httpLoad("doc?method=updateDocument").indexOf("<b>binary</b>") != -1);
+
+ String enc = Base64.encodeBase64String(b);
+ send("deleteAllDocuments", "{indexName: index}");
+ String params = "{indexName: index, binary: {content: '" + enc + "', fileName: 'test.docx', mappings: {body: body, Author: author, 'Character Count': charCount}}}";
+ long gen = getLong(send("addDocument", params), "indexGen");
+
+ JSONObject result = send("search", "{indexName: index, searcher: {indexGen: " + gen + "}, retrieveFields: [author, charCount], queryText: 'body:wee'}");
+ assertEquals(1, getInt(result, "totalHits"));
+ assertEquals("Michael McCandless", getString(result, "hits[0].fields.author"));
+ assertEquals(38, getInt(result, "hits[0].fields.charCount"));
+ }
+
+ public void testBulkAddWordDoc() throws Exception {
+ send("deleteAllDocuments", "{indexName: index}");
+
+ // nocommit get test-resources onto classpath so I don't have to
+ // hardwire full path!
+ RandomAccessFile f = new RandomAccessFile("../../../src/test-resources/test.docx", "r");
+ byte[] b = new byte[(int)f.length()];
+ f.read(b);
+ String enc = Base64.encodeBase64String(b);
+
+ // Make sure doc is updated w/ new binary param:
+ assertTrue(httpLoad("doc?method=bulkAddDocument").indexOf("<b>binary</b>") != -1);
+ assertTrue(httpLoad("doc?method=bulkAddDocuments").indexOf("<b>binary</b>") != -1);
+ assertTrue(httpLoad("doc?method=bulkUpdateDocument").indexOf("<b>binary</b>") != -1);
+
+ StringBuilder sb = new StringBuilder();
+ sb.append("{\"indexName\": \"index\", \"documents\": [");
+ for(int i=0;i<100;i++) {
+ if (i > 0) {
+ sb.append(',');
+ }
+ sb.append("{\"binary\": {\"content\": \"" + enc + "\", \"fileName\": \"test.docx\", \"mappings\": {\"body\": \"body\", \"Author\": \"author\", \"Character Count\": \"charCount\"}}}");
+ }
+ sb.append("]}");
+ String s = sb.toString();
+
+ JSONObject result = sendChunked(s, "bulkAddDocument");
+ long indexGen = getLong(result, "indexGen");
+ result = send("search", "{indexName: index, queryText: wee, searcher: {indexGen: " + indexGen + "}}");
+ assertEquals(100, result.get("totalHits"));
+ }
+}
Added: lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/test.py
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/test.py?rev=1553272&view=auto
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/test.py (added)
+++ lucene/dev/branches/lucene5376/lucene/server/plugins/BinaryDocument/test.py Tue Dec 24 13:39:22 2013
@@ -0,0 +1,8 @@
+import sys
+sys.path.insert(0, '../../src/python')
+import os
+import util
+
+util.run('ant compile-test', '../..')
+util.run('ant jar test')
+
Added: lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/BlendedComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/BlendedComparator.java?rev=1553272&view=auto
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/BlendedComparator.java (added)
+++ lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/BlendedComparator.java Tue Dec 24 13:39:22 2013
@@ -0,0 +1,96 @@
+package org.apache.lucene.server;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.ScoreCachingWrappingScorer;
+import org.apache.lucene.search.Scorer;
+
+/** Blends score with another source; this sorts in reverse
+ * float order, i.e. higher scores are better. */
+
+abstract class BlendedComparator extends FieldComparator<Float> {
+
+ private final float[] scores;
+ private float bottom;
+ protected Scorer scorer;
+
+ public BlendedComparator(int numHits) {
+ scores = new float[numHits];
+ }
+
+ @Override
+ public int compare(int slot1, int slot2) {
+ return Float.compare(scores[slot2], scores[slot1]);
+ }
+
+ @Override
+ public int compareBottom(int doc) {
+ return Float.compare(getScore(doc), bottom);
+ }
+
+ @Override
+ public void copy(int slot, int doc) throws IOException {
+ scores[slot] = getScore(doc);
+ }
+
+ @Override
+ public FieldComparator<Float> setNextReader(AtomicReaderContext context) throws IOException {
+ return this;
+ }
+
+ @Override
+ public void setBottom(final int bottom) {
+ this.bottom = scores[bottom];
+ }
+
+ @Override
+ public void setScorer(Scorer scorer) {
+ // wrap with a ScoreCachingWrappingScorer so that successive calls to
+ // score() will not incur score computation over and
+ // over again.
+ if (!(scorer instanceof ScoreCachingWrappingScorer)) {
+ this.scorer = new ScoreCachingWrappingScorer(scorer);
+ } else {
+ this.scorer = scorer;
+ }
+ }
+
+ @Override
+ public Float value(int slot) {
+ return Float.valueOf(scores[slot]);
+ }
+
+ // Override because we sort reverse of natural Float order:
+ @Override
+ public int compareValues(Float first, Float second) {
+ // Reversed intentionally because relevance by default
+ // sorts descending:
+ return second.compareTo(first);
+ }
+
+ @Override
+ public int compareDocToValue(int doc, Float valueObj) throws IOException {
+ return Float.compare(valueObj.floatValue(), getScore(doc));
+ }
+
+ protected abstract float getScore(int doc);
+}
Added: lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/BlockingThreadPoolExecutor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/BlockingThreadPoolExecutor.java?rev=1553272&view=auto
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/BlockingThreadPoolExecutor.java (added)
+++ lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/BlockingThreadPoolExecutor.java Tue Dec 24 13:39:22 2013
@@ -0,0 +1,61 @@
+package org.apache.lucene.server;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+// From Brian Goetz, JCiP:
+class BlockingThreadPoolExecutor extends ThreadPoolExecutor {
+
+ private final Semaphore semaphore;
+
+ public BlockingThreadPoolExecutor(int bound, int corePoolSize, int maxPoolSize, long keepAliveTime, TimeUnit unit, BlockingQueue<Runnable> queue, ThreadFactory threadFactory) {
+ super(corePoolSize, maxPoolSize, keepAliveTime, unit, queue, threadFactory);
+ this.semaphore = new Semaphore(bound);
+ }
+
+ @Override
+ public void execute(Runnable task) {
+ try {
+ semaphore.acquire();
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(ie);
+ }
+
+ try {
+ super.execute(task);
+ } catch(RuntimeException e) {
+ // specifically, handle RejectedExecutionException
+ semaphore.release();
+ throw e;
+ } catch(Error e) {
+ semaphore.release();
+ throw e;
+ }
+ }
+
+ @Override
+ protected void afterExecute(Runnable r, Throwable t) {
+ semaphore.release();
+ }
+}
Added: lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/Constants.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/Constants.java?rev=1553272&view=auto
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/Constants.java (added)
+++ lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/Constants.java Tue Dec 24 13:39:22 2013
@@ -0,0 +1,28 @@
+package org.apache.lucene.server;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.regex.Pattern;
+
+public class Constants {
+ private Constants() {
+ }
+
+ public static final char INFORMATION_SEP = '\u001f';
+ public static final String INFORMATION_SEP_REGEX = Pattern.quote(Character.toString(INFORMATION_SEP));
+}