You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sa...@apache.org on 2011/06/15 00:20:04 UTC
svn commit: r1135827 - in /lucene/dev/branches/branch_3x: ./
dev-tools/maven/ lucene/ lucene/contrib/ lucene/lib/ solr/
Author: sarowe
Date: Tue Jun 14 22:20:03 2011
New Revision: 1135827
URL: http://svn.apache.org/viewvc?rev=1135827&view=rev
Log:
LUCENE-3204: Include maven-ant-tasks jar in the source tree and use this jar from generate-maven-artifacts
Added:
lucene/dev/branches/branch_3x/lucene/lib/maven-ant-tasks-2.1.1.jar
- copied unchanged from r1135801, lucene/dev/trunk/lucene/lib/maven-ant-tasks-2.1.1.jar
lucene/dev/branches/branch_3x/lucene/lib/maven-ant-tasks-LICENSE-ASL.txt
- copied unchanged from r1135801, lucene/dev/trunk/lucene/lib/maven-ant-tasks-LICENSE-ASL.txt
lucene/dev/branches/branch_3x/lucene/lib/maven-ant-tasks-NOTICE.txt
- copied unchanged from r1135818, lucene/dev/trunk/lucene/lib/maven-ant-tasks-NOTICE.txt
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/dev-tools/maven/README.maven
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/CHANGES.txt
lucene/dev/branches/branch_3x/lucene/build.xml
lucene/dev/branches/branch_3x/lucene/common-build.xml
lucene/dev/branches/branch_3x/lucene/contrib/contrib-build.xml
lucene/dev/branches/branch_3x/solr/ (props changed)
lucene/dev/branches/branch_3x/solr/CHANGES.txt
lucene/dev/branches/branch_3x/solr/build.xml
lucene/dev/branches/branch_3x/solr/common-build.xml
Modified: lucene/dev/branches/branch_3x/dev-tools/maven/README.maven
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/dev-tools/maven/README.maven?rev=1135827&r1=1135826&r2=1135827&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/dev-tools/maven/README.maven (original)
+++ lucene/dev/branches/branch_3x/dev-tools/maven/README.maven Tue Jun 14 22:20:03 2011
@@ -21,18 +21,7 @@ A. How to use nightly Jenkins-built Luce
B. How to generate Lucene Maven artifacts
- 1. Prerequisites: JDK 1.5+, Ant 1.7.X, and maven-ant-tasks-2.1.1.jar
-
- In order to generate Maven artifacts for Lucene/Solr, you must first
- download the Maven ant tasks JAR (maven-ant-tasks-2.1.1.jar), e.g.
- from <http://maven.apache.org/ant-tasks/download.html>, and add it
- to any one of the following:
-
- a. Your $HOME/.ant/lib/ directory (C:\Users\username\.ant\lib\ under
- Windows Vista/7); or
- b. Your $ANT_HOME/lib/ directory (%ANT_HOME%\lib\ under Windows); or
- c. Your $CLASSPATH (%CLASSPATH% under Windows); or
- d. Your ant commond line: "-lib /path/to/maven-ant-tasks-2.1.1.jar".
+ 1. Prerequisites: JDK 1.5+ and Ant 1.7.X
2. Run the following command from the lucene/ directory:
@@ -46,8 +35,7 @@ B. How to generate Lucene Maven artifact
C. How to generate Solr Maven artifacts
- 1. Prerequisites: JDK 1.5+; Ant 1.7.X; and maven-ant-tasks-2.1.1.jar
- (see item A.1. above for where to put the Maven ant tasks jar).
+ 1. Prerequisites: JDK 1.5+ and Ant 1.7.X
2. Run the following from the solr/ directory:
Modified: lucene/dev/branches/branch_3x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/CHANGES.txt?rev=1135827&r1=1135826&r2=1135827&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/CHANGES.txt Tue Jun 14 22:20:03 2011
@@ -3,6 +3,468 @@ Lucene Change Log
For more information on past and future Lucene versions, please see:
http://s.apache.org/luceneversions
+======================= Trunk (not yet released) =======================
+
+Changes in backwards compatibility policy
+
+* LUCENE-1458, LUCENE-2111, LUCENE-2354: Changes from flexible indexing:
+
+ - On upgrading to 3.1, if you do not fully reindex your documents,
+ Lucene will emulate the new flex API on top of the old index,
+ incurring some performance cost (up to ~10% slowdown, typically).
+ To prevent this slowdown, use oal.index.IndexUpgrader
+ to upgrade your indexes to latest file format (LUCENE-3082).
+
+ Mixed flex/pre-flex indexes are perfectly fine -- the two
+ emulation layers (flex API on pre-flex index, and pre-flex API on
+ flex index) will remap the access as required. So on upgrading to
+ 3.1 you can start indexing new documents into an existing index.
+ To get optimal performance, use oal.index.IndexUpgrader
+ to upgrade your indexes to latest file format (LUCENE-3082).
+
+ - The postings APIs (TermEnum, TermDocsEnum, TermPositionsEnum)
+ have been removed in favor of the new flexible
+ indexing (flex) APIs (Fields, FieldsEnum, Terms, TermsEnum,
+ DocsEnum, DocsAndPositionsEnum). One big difference is that field
+ and terms are now enumerated separately: a TermsEnum provides a
+ BytesRef (wraps a byte[]) per term within a single field, not a
+ Term. Another is that when asking for a Docs/AndPositionsEnum, you
+ now specify the skipDocs explicitly (typically this will be the
+ deleted docs, but in general you can provide any Bits).
+
+ - MultiReader ctor now throws IOException
+
+ - Directory.copy/Directory.copyTo now copies all files (not just
+ index files), since what is and isn't and index file is now
+ dependent on the codecs used.
+
+ - UnicodeUtil now uses BytesRef for UTF-8 output, and some method
+ signatures have changed to CharSequence. These are internal APIs
+ and subject to change suddenly.
+
+ - Positional queries (PhraseQuery, *SpanQuery) will now throw an
+ exception if use them on a field that omits positions during
+ indexing (previously they silently returned no results).
+
+ - FieldCache.{Byte,Short,Int,Long,Float,Double}Parser's API has
+ changed -- each parse method now takes a BytesRef instead of a
+ String. If you have an existing Parser, a simple way to fix it is
+ invoke BytesRef.utf8ToString, and pass that String to your
+ existing parser. This will work, but performance would be better
+ if you could fix your parser to instead operate directly on the
+ byte[] in the BytesRef.
+
+ - The internal (experimental) API of NumericUtils changed completely
+ from String to BytesRef. Client code should never use this class,
+ so the change would normally not affect you. If you used some of
+ the methods to inspect terms or create TermQueries out of
+ prefix encoded terms, change to use BytesRef. Please note:
+ Do not use TermQueries to search for single numeric terms.
+ The recommended way is to create a corresponding NumericRangeQuery
+ with upper and lower bound equal and included. TermQueries do not
+ score correct, so the constant score mode of NRQ is the only
+ correct way to handle single value queries.
+
+ - NumericTokenStream now works directly on byte[] terms. If you
+ plug a TokenFilter on top of this stream, you will likely get
+ an IllegalArgumentException, because the NTS does not support
+ TermAttribute/CharTermAttribute. If you want to further filter
+ or attach Payloads to NTS, use the new NumericTermAttribute.
+
+ (Mike McCandless, Robert Muir, Uwe Schindler, Mark Miller, Michael Busch)
+
+* LUCENE-2265: FuzzyQuery and WildcardQuery now operate on Unicode codepoints,
+ not unicode code units. For example, a Wildcard "?" represents any unicode
+ character. Furthermore, the rest of the automaton package and RegexpQuery use
+ true Unicode codepoint representation. (Robert Muir, Mike McCandless)
+
+* LUCENE-2380: The String-based FieldCache methods (getStrings,
+ getStringIndex) have been replaced with BytesRef-based equivalents
+ (getTerms, getTermsIndex). Also, the sort values (returned in
+ FieldDoc.fields) when sorting by SortField.STRING or
+ SortField.STRING_VAL are now BytesRef instances. See MIGRATE.txt
+ for more details. (yonik, Mike McCandless)
+
+* LUCENE-2480: Though not a change in backwards compatibility policy, pre-3.0
+ indexes are no longer supported. You should upgrade to 3.x first, then run
+ optimize(), or reindex. (Shai Erera, Earwin Burrfoot)
+
+* LUCENE-2484: Removed deprecated TermAttribute. Use CharTermAttribute
+ and TermToBytesRefAttribute instead. (Uwe Schindler)
+
+* LUCENE-2600: Remove IndexReader.isDeleted in favor of
+ IndexReader.getDeletedDocs(). (Mike McCandless)
+
+* LUCENE-2667: FuzzyQuery's defaults have changed for more performant
+ behavior: the minimum similarity is 2 edit distances from the word,
+ and the priority queue size is 50. To support this, FuzzyQuery now allows
+ specifying unscaled edit distances (foobar~2). If your application depends
+ upon the old defaults of 0.5 (scaled) minimum similarity and Integer.MAX_VALUE
+ priority queue size, you can use FuzzyQuery(Term, float, int, int) to specify
+ those explicitly.
+
+* LUCENE-2674: MultiTermQuery.TermCollector.collect now accepts the
+ TermsEnum as well. (Robert Muir, Mike McCandless)
+
+* LUCENE-588: WildcardQuery and QueryParser now allows escaping with
+ the '\' character. Previously this was impossible (you could not escape */?,
+ for example). If your code somehow depends on the old behavior, you will
+ need to change it (e.g. using "\\" to escape '\' itself).
+ (Sunil Kamath, Terry Yang via Robert Muir)
+
+* LUCENE-2771: IndexReader.norms() now throws UOE on non-atomic IndexReaders. If
+ you really want a top-level norms, use MultiNorms or SlowMultiReaderWrapper.
+ (Uwe Schindler, Robert Muir)
+
+* LUCENE-2837: Collapsed Searcher, Searchable into IndexSearcher;
+ removed contrib/remote and MultiSearcher (Mike McCandless); absorbed
+ ParallelMultiSearcher into IndexSearcher as an optional
+ ExecutorServiced passed to its ctor. (Mike McCandless)
+
+* LUCENE-2908: Removed serialization code from lucene classes. It is recommended
+ that you serialize user search needs at a higher level in your application.
+ (Robert Muir)
+
+* LUCENE-2831: Changed Weight#scorer, Weight#explain & Filter#getDocIdSet to
+ operate on a AtomicReaderContext instead of directly on IndexReader to enable
+ searches to be aware of IndexSearcher's context. (Simon Willnauer)
+
+* LUCENE-2839: Scorer#score(Collector,int,int) is now public because it is
+ called from other classes and part of public API. (Uwe Schindler)
+
+* LUCENE-2865: Weight#scorer(AtomicReaderContext, boolean, boolean) now accepts
+ a ScorerContext struct instead of booleans.(Simon Willnauer)
+
+* LUCENE-2882: Cut over SpanQuery#getSpans to AtomicReaderContext to enforce
+ per segment semantics on SpanQuery & Spans. (Simon Willnauer)
+
+* LUCENE-2236: Similarity can now be configured on a per-field basis. See the
+ migration notes in MIGRATE.txt for more details. (Robert Muir, Doron Cohen)
+
+* LUCENE-2315: AttributeSource's methods for accessing attributes are now final,
+ else its easy to corrupt the internal states. (Uwe Schindler)
+
+* LUCENE-2814: The IndexWriter.flush method no longer takes "boolean
+ flushDocStores" argument, as we now always flush doc stores (index
+ files holding stored fields and term vectors) while flushing a
+ segment. (Mike McCandless)
+
+Changes in Runtime Behavior
+
+* LUCENE-2846: omitNorms now behaves like omitTermFrequencyAndPositions, if you
+ omitNorms(true) for field "a" for 1000 documents, but then add a document with
+ omitNorms(false) for field "a", all documents for field "a" will have no norms.
+ Previously, Lucene would fill the first 1000 documents with "fake norms" from
+ Similarity.getDefault(). (Robert Muir, Mike Mccandless)
+
+* LUCENE-2846: When some documents contain field "a", and others do not, the
+ documents that don't have the field get a norm byte value of 0. Previously, Lucene
+ would populate "fake norms" with Similarity.getDefault() for these documents.
+ (Robert Muir, Mike Mccandless)
+
+* LUCENE-2720: IndexWriter throws IndexFormatTooOldException on open, rather
+ than later when e.g. a merge starts. (Shai Erera, Mike McCandless, Uwe Schindler)
+
+* LUCENE-2881: FieldInfos is now tracked per segment. Before it was tracked
+ per IndexWriter session, which resulted in FieldInfos that had the FieldInfo
+ properties from all previous segments combined. Field numbers are now tracked
+ globally across IndexWriter sessions and persisted into a X.fnx file on
+ successful commit. The corresponding file format changes are backwards-
+ compatible. (Michael Busch, Simon Willnauer)
+
+* LUCENE-2956, LUCENE-2573, LUCENE-2324, LUCENE-2555: Changes from
+ DocumentsWriterPerThread:
+
+ - IndexWriter now uses a DocumentsWriter per thread when indexing documents.
+ Each DocumentsWriterPerThread indexes documents in its own private segment,
+ and the in memory segments are no longer merged on flush. Instead, each
+ segment is separately flushed to disk and subsequently merged with normal
+ segment merging.
+
+ - DocumentsWriterPerThread (DWPT) is now flushed concurrently based on a
+ FlushPolicy. When a DWPT is flushed, a fresh DWPT is swapped in so that
+ indexing may continue concurrently with flushing. The selected
+ DWPT flushes all its RAM resident documents do disk. Note: Segment flushes
+ don't flush all RAM resident documents but only the documents private to
+ the DWPT selected for flushing.
+
+ - Flushing is now controlled by FlushPolicy that is called for every add,
+ update or delete on IndexWriter. By default DWPTs are flushed either on
+ maxBufferedDocs per DWPT or the global active used memory. Once the active
+ memory exceeds ramBufferSizeMB only the largest DWPT is selected for
+ flushing and the memory used by this DWPT is substracted from the active
+ memory and added to a flushing memory pool, which can lead to temporarily
+ higher memory usage due to ongoing indexing.
+
+ - IndexWriter now can utilize ramBufferSize > 2048 MB. Each DWPT can address
+ up to 2048 MB memory such that the ramBufferSize is now bounded by the max
+ number of DWPT avaliable in the used DocumentsWriterPerThreadPool.
+ IndexWriters net memory consumption can grow far beyond the 2048 MB limit if
+ the applicatoin can use all available DWPTs. To prevent a DWPT from
+ exhausting its address space IndexWriter will forcefully flush a DWPT if its
+ hard memory limit is exceeded. The RAMPerThreadHardLimitMB can be controlled
+ via IndexWriterConfig and defaults to 1945 MB.
+ Since IndexWriter flushes DWPT concurrently not all memory is released
+ immediately. Applications should still use a ramBufferSize significantly
+ lower than the JVMs avaliable heap memory since under high load multiple
+ flushing DWPT can consume substantial transient memory when IO performance
+ is slow relative to indexing rate.
+
+ - IndexWriter#commit now doesn't block concurrent indexing while flushing all
+ 'currently' RAM resident documents to disk. Yet, flushes that occur while a
+ a full flush is running are queued and will happen after all DWPT involved
+ in the full flush are done flushing. Applications using multiple threads
+ during indexing and trigger a full flush (eg call commmit() or open a new
+ NRT reader) can use significantly more transient memory.
+
+ - IndexWriter#addDocument and IndexWriter.updateDocument can block indexing
+ threads if the number of active + number of flushing DWPT exceed a
+ safety limit. By default this happens if 2 * max number available thread
+ states (DWPTPool) is exceeded. This safety limit prevents applications from
+ exhausting their available memory if flushing can't keep up with
+ concurrently indexing threads.
+
+ - IndexWriter only applies and flushes deletes if the maxBufferedDelTerms
+ limit is reached during indexing. No segment flushes will be triggered
+ due to this setting.
+
+ - IndexWriter#flush(boolean, boolean) doesn't synchronized on IndexWriter
+ anymore. A dedicated flushLock has been introduced to prevent multiple full-
+ flushes happening concurrently.
+
+ - DocumentsWriter doesn't write shared doc stores anymore.
+
+ (Mike McCandless, Michael Busch, Simon Willnauer)
+
+* LUCENE-3146: IndexReader.setNorm throws IllegalStateException if the field
+ does not store norms. (Shai Erera, Mike McCandless)
+
+API Changes
+
+* LUCENE-2302, LUCENE-1458, LUCENE-2111, LUCENE-2514: Terms are no longer
+ required to be character based. Lucene views a term as an arbitrary byte[]:
+ during analysis, character-based terms are converted to UTF8 byte[],
+ but analyzers are free to directly create terms as byte[]
+ (NumericField does this, for example). The term data is buffered as
+ byte[] during indexing, written as byte[] into the terms dictionary,
+ and iterated as byte[] (wrapped in a BytesRef) by IndexReader for
+ searching.
+
+* LUCENE-1458, LUCENE-2111: IndexReader now directly exposes its
+ deleted docs (getDeletedDocs), providing a new Bits interface to
+ directly query by doc ID.
+
+* LUCENE-2691: IndexWriter.getReader() has been made package local and is now
+ exposed via open and reopen methods on IndexReader. The semantics of the
+ call is the same as it was prior to the API change.
+ (Grant Ingersoll, Mike McCandless)
+
+* LUCENE-2566: QueryParser: Unary operators +,-,! will not be treated as
+ operators if they are followed by whitespace. (yonik)
+
+* LUCENE-2831: Weight#scorer, Weight#explain, Filter#getDocIdSet,
+ Collector#setNextReader & FieldComparator#setNextReader now expect an
+ AtomicReaderContext instead of an IndexReader. (Simon Willnauer)
+
+* LUCENE-2846: Remove the deprecated IndexReader.setNorm(int, String, float).
+ This method was only syntactic sugar for setNorm(int, String, byte), but
+ using the global Similarity.getDefault().encodeNormValue. Use the byte-based
+ method instead to ensure that the norm is encoded with your Similarity.
+ Also removed norms(String, byte[], int), which was only used by MultiReader
+ for building top-level norms. If you really need a top-level norms, use
+ MultiNorms or SlowMultiReaderWrapper. (Robert Muir, Mike Mccandless)
+
+* LUCENE-2892: Add QueryParser.newFieldQuery (called by getFieldQuery by default)
+ which takes Analyzer as a parameter, for easier customization by subclasses.
+ (Robert Muir)
+
+* LUCENE-2953: In addition to changes in 3.x, PriorityQueue#initialize(int)
+ function was moved into the ctor. (Uwe Schindler, Yonik Seeley)
+
+New features
+
+* LUCENE-2604: Added RegexpQuery support to QueryParser. Regular expressions
+ are directly supported by the standard queryparser.
+ (Simon Willnauer, Robert Muir)
+
+* LUCENE-1606, LUCENE-2089: Adds AutomatonQuery, a MultiTermQuery that
+ matches terms against a finite-state machine. Implement WildcardQuery
+ and FuzzyQuery with finite-state methods. Adds RegexpQuery.
+ (Robert Muir, Mike McCandless, Uwe Schindler, Mark Miller)
+
+* LUCENE-1990: Adds internal packed ints implementation, to be used
+ for more efficient storage of int arrays when the values are
+ bounded, for example for storing the terms dict index (Toke
+ Eskildsen via Mike McCandless)
+
+* LUCENE-2321: Cutover to a more RAM efficient packed-ints based
+ representation for the in-memory terms dict index. (Mike
+ McCandless)
+
+* LUCENE-2126: Add new classes for data (de)serialization: DataInput
+ and DataOutput. IndexInput and IndexOutput extend these new classes.
+ (Michael Busch)
+
+* LUCENE-1458, LUCENE-2111: With flexible indexing it is now possible
+ for an application to create its own postings codec, to alter how
+ fields, terms, docs and positions are encoded into the index. The
+ standard codec is the default codec. Both IndexWriter and
+ IndexReader accept a CodecProvider class to obtain codecs for newly
+ written segments as well as existing segments opened for reading.
+
+* LUCENE-1458, LUCENE-2111: Some experimental codecs have been added
+ for flexible indexing, including pulsing codec (inlines
+ low-frequency terms directly into the terms dict, avoiding seeking
+ for some queries), sep codec (stores docs, freqs, positions, skip
+ data and payloads in 5 separate files instead of the 2 used by
+ standard codec), and int block (really a "base" for using
+ block-based compressors like PForDelta for storing postings data).
+
+* LUCENE-1458, LUCENE-2111: The in-memory terms index used by standard
+ codec is more RAM efficient: terms data is stored as block byte
+ arrays and packed integers. Net RAM reduction for indexes that have
+ many unique terms should be substantial, and initial open time for
+ IndexReaders should be faster. These gains only apply for newly
+ written segments after upgrading.
+
+* LUCENE-1458, LUCENE-2111: Terms data are now buffered directly as
+ byte[] during indexing, which uses half the RAM for ascii terms (and
+ also numeric fields). This can improve indexing throughput for
+ applications that have many unique terms, since it reduces how often
+ a new segment must be flushed given a fixed RAM buffer size.
+
+* LUCENE-2489: Added PerFieldCodecWrapper (in oal.index.codecs) which
+ lets you set the Codec per field (Mike McCandless)
+
+* LUCENE-2373: Extend CodecProvider to use SegmentInfosWriter and
+ SegmentInfosReader to allow customization of SegmentInfos data.
+ (Andrzej Bialecki)
+
+* LUCENE-2504: FieldComparator.setNextReader now returns a
+ FieldComparator instance. You can "return this", to just reuse the
+ same instance, or you can return a comparator optimized to the new
+ segment. (yonik, Mike McCandless)
+
+* LUCENE-2648: PackedInts.Iterator now supports to advance by more than a
+ single ordinal. (Simon Willnauer)
+
+* LUCENE-2649: Objects in the FieldCache can optionally store Bits
+ that mark which docs have real values in the native[] (ryan)
+
+* LUCENE-2664: Add SimpleText codec, which stores all terms/postings
+ data in a single text file for transparency (at the expense of poor
+ performance). (Sahin Buyrukbilen via Mike McCandless)
+
+* LUCENE-2589: Add a VariableSizedIntIndexInput, which, when used w/
+ Sep*, makes it simple to take any variable sized int block coders
+ (like Simple9/16) and use them in a codec. (Mike McCandless)
+
+* LUCENE-2597: Add oal.index.SlowMultiReaderWrapper, to wrap a
+ composite reader (eg MultiReader or DirectoryReader), making it
+ pretend it's an atomic reader. This is a convenience class (you can
+ use MultiFields static methods directly, instead) if you need to use
+ the flex APIs directly on a composite reader. (Mike McCandless)
+
+* LUCENE-2690: MultiTermQuery boolean rewrites per segment.
+ (Uwe Schindler, Robert Muir, Mike McCandless, Simon Willnauer)
+
+* LUCENE-996: The QueryParser now accepts mixed inclusive and exclusive
+ bounds for range queries. Example: "{3 TO 5]"
+ QueryParser subclasses that overrode getRangeQuery will need to be changed
+ to use the new getRangeQuery method. (Andrew Schurman, Mark Miller, yonik)
+
+* LUCENE-995: The QueryParser now interprets * as an open end for range
+ queries. Literal asterisks may be represented by quoting or escaping
+ (i.e. \* or "*") Custom QueryParser subclasses overriding getRangeQuery()
+ will be passed null for any open endpoint. (Adriano Crestani, yonik)
+
+* LUCENE-2742: Add native per-field codec support. CodecProvider lets you now
+ register a codec for each field and which is in turn recorded in the segment
+ and field information. Codecs are maintained on a per-segment basis and be
+ resolved without knowing the actual codec used for writing the segment.
+ (Simon Willnauer)
+
+* LUCENE-2741: Add support for multiple codecs that use the same file
+ extensions within the same segment. Codecs now use their per-segment codec
+ ID in the file names. (Simon Willnauer)
+
+* LUCENE-2843: Added a new terms index impl,
+ VariableGapTermsIndexWriter/Reader, that accepts a pluggable
+ IndexTermSelector for picking which terms should be indexed in the
+ terms dict. This impl stores the indexed terms in an FST, which is
+ much more RAM efficient than FixedGapTermsIndex. (Mike McCandless)
+
+* LUCENE-2862: Added TermsEnum.totalTermFreq() and
+ Terms.getSumTotalTermFreq(). (Mike McCandless, Robert Muir)
+
+* LUCENE-3003: Added new expert class oal.index.DocTermsOrd,
+ refactored from Solr's UnInvertedField, for accessing term ords for
+ multi-valued fields, per document. This is similar to FieldCache in
+ that it inverts the index to compute the ords, but differs in that
+ it's able to handle multi-valued fields and does not hold the term
+ bytes in RAM. (Mike McCandless)
+
+* LUCENE-3108, LUCENE-2935, LUCENE-2168, LUCENE-1231: Changes from
+ IndexDocValues (ColumnStrideFields):
+
+ - IndexWriter now supports typesafe dense per-document values stored in
+ a column like storage. IndexDocValues are stored on a per-document
+ basis where each documents field can hold exactly one value of a given
+ type. IndexDocValues are provided via Fieldable and can be used in
+ conjunction with stored and indexed values.
+
+ - IndexDocValues provides an entirely RAM resident document id to value
+ mapping per field as well as a DocIdSetIterator based disk-resident
+ sequential access API relying on filesystem-caches.
+
+ - Both APIs are exposed via IndexReader and the Codec / Flex API allowing
+ expert users to integrate customized IndexDocValues reader and writer
+ implementations by extending existing Codecs.
+
+ - IndexDocValues provides implementations for primitive datatypes like int,
+ long, float, double and arrays of byte. Byte based implementations further
+ provide storage variants like straight or dereferenced stored bytes, fixed
+ and variable length bytes as well as index time sorted based on user-provided
+ comparators.
+
+ (Mike McCandless, Simon Willnauer)
+
+Optimizations
+
+* LUCENE-2588: Don't store unnecessary suffixes when writing the terms
+ index, saving RAM in IndexReader; change default terms index
+ interval from 128 to 32, because the terms index now requires much
+ less RAM. (Robert Muir, Mike McCandless)
+
+* LUCENE-2669: Optimize NumericRangeQuery.NumericRangeTermsEnum to
+ not seek backwards when a sub-range has no terms. It now only seeks
+ when the current term is less than the next sub-range's lower end.
+ (Uwe Schindler, Mike McCandless)
+
+* LUCENE-2694: Optimize MultiTermQuery to be single pass for Term lookups.
+ MultiTermQuery now stores TermState per leaf reader during rewrite to re-
+ seek the term dictionary in TermQuery / TermWeight.
+ (Simon Willnauer, Mike McCandless, Robert Muir)
+
+* LUCENE-2897: Apply deleted terms while flushing a segment. We still
+ buffer deleted terms to later apply to past segments. (Mike McCandless)
+
+* LUCENE-1736: DateTools.java general improvements.
+ (David Smiley via Steve Rowe)
+
+Bug fixes
+
+* LUCENE-2633: PackedInts Packed32 and Packed64 did not support internal
+ structures larger than 256MB (Toke Eskildsen via Mike McCandless)
+
+* LUCENE-2803: The FieldCache can miss values if an entry for a reader
+ with more document deletions is requested before a reader with fewer
+ deletions, provided they share some segments. (yonik)
+
+* LUCENE-2645: Fix false assertion error when same token was added one
+ after another with 0 posIncr. (David Smiley, Kurosaka Teruhiko via Mike
+ McCandless)
+
======================= Lucene 3.x (not yet released) ================
Changes in backwards compatibility policy
@@ -17,7 +479,7 @@ Changes in runtime behavior
lock file is not stored in the index has changed. This means you
will see a different lucene-XXX-write.lock in your lock directory.
(Robert Muir, Uwe Schindler, Mike McCandless)
-
+
* LUCENE-3146: IndexReader.setNorm throws IllegalStateException if the field
does not store norms. (Shai Erera, Mike McCandless)
@@ -49,11 +511,6 @@ Bug fixes
ArrayIndexOutOfBoundsException (selckin, Robert Muir, Mike
McCandless)
-Optimizations
-
-* LUCENE-1736: DateTools.java general improvements.
- (David Smiley via Steve Rowe)
-
New Features
* LUCENE-3140: Added experimental FST implementation to Lucene.
@@ -67,6 +524,16 @@ Build
* LUCENE-1344: Create OSGi bundle using dev-tools/maven.
(Nicolas Lalevée, Luca Stancapiano via ryan)
+
+* LUCENE-3204: The maven-ant-tasks jar is now included in the source tree;
+ users of the generate-maven-artifacts target no longer have to manually
+ place this jar in the Ant classpath. NOTE: when Ant looks for the
+ maven-ant-tasks jar, it looks first in its pre-existing classpath, so
+ any copies it finds will be used instead of the copy included in the
+ Lucene/Solr source tree. For this reason, it is recommeded to remove
+ any copies of the maven-ant-tasks jar in the Ant classpath, e.g. under
+ ~/.ant/lib/ or under the Ant installation's lib/ directory. (Steve Rowe)
+
======================= Lucene 3.2.0 =======================
@@ -154,7 +621,7 @@ Optimizations
* LUCENE-3126: IndexWriter.addIndexes copies incoming segments into CFS if they
aren't already and MergePolicy allows that. (Shai Erera)
-
+
Bug fixes
* LUCENE-2996: addIndexes(IndexReader) did not flush before adding the new
@@ -197,7 +664,7 @@ Test Cases
* LUCENE-3002: added 'tests.iter.min' to control 'tests.iter' by allowing to
stop iterating if at least 'tests.iter.min' ran and a failure occured.
(Shai Erera, Chris Hostetter)
-
+
======================= Lucene 3.1.0 =======================
Changes in backwards compatibility policy
@@ -1193,6 +1660,9 @@ Bug fixes
files when a mergedSegmentWarmer is set on IndexWriter. (Mike
McCandless)
+* LUCENE-2130: Fix performance issue when FuzzyQuery runs on a
+ multi-segment index (Michael McCandless)
+
API Changes
* LUCENE-2281: added doBeforeFlush to IndexWriter to allow extensions to perform
Modified: lucene/dev/branches/branch_3x/lucene/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/build.xml?rev=1135827&r1=1135826&r2=1135827&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/build.xml (original)
+++ lucene/dev/branches/branch_3x/lucene/build.xml Tue Jun 14 22:20:03 2011
@@ -461,7 +461,10 @@
</target>
- <target name="generate-maven-artifacts" depends="maven.ant.tasks-check, package, jar-src, jar-test-framework-src, javadocs">
+ <target name="generate-maven-artifacts" depends="package, jar-src, jar-test-framework-src, javadocs">
+ <taskdef resource="org/apache/maven/artifact/ant/antlib.xml"
+ uri="antlib:org.apache.maven.artifact.ant"
+ classpathref="maven-ant-tasks.classpath"/>
<sequential>
<ant target="get-maven-poms" dir=".."/>
Modified: lucene/dev/branches/branch_3x/lucene/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/common-build.xml?rev=1135827&r1=1135826&r2=1135827&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/common-build.xml (original)
+++ lucene/dev/branches/branch_3x/lucene/common-build.xml Tue Jun 14 22:20:03 2011
@@ -64,6 +64,11 @@
<pathelement location="${common.dir}/build/classes/tools"/>
</path>
+ <path id="maven-ant-tasks.classpath">
+ <fileset dir="${common.dir}/lib">
+ <include name="maven-ant-tasks-*.jar"/>
+ </fileset>
+ </path>
<!-- default arguments to pass to JVM executing tests -->
<property name="testmethod" value=""/>
@@ -189,11 +194,6 @@
</and>
</condition>
- <available
- property="maven.ant.tasks.present"
- classname="org.apache.maven.artifact.ant.Pom"
- />
-
<target name="clean"
description="Removes contents of build and dist directories">
<delete dir="${build.dir}"/>
@@ -315,20 +315,6 @@
<jarify/>
</target>
- <target name="maven.ant.tasks-check">
- <fail unless="maven.ant.tasks.present">#
- ##########################################################################
- Maven ant tasks not found.
-
- Please download the Maven ant tasks JAR (maven-ant-tasks-2.1.1.jar)
- from http://maven.apache.org/ant-tasks/download.html and add it to your
- $$HOME/.ant/lib/ directory, or to your $$ANT_HOME/lib/ directory, or
- to your $$CLASSPATH, or add "-lib /path/to/maven-ant-tasks-2.1.1.jar"
- to the ant command.
- ##########################################################################
- </fail>
- </target>
-
<macrodef name="m2-deploy" description="Builds a Maven artifact">
<element name="artifact-attachments" optional="yes"/>
<attribute name="pom.xml" default="pom.xml"/>
Modified: lucene/dev/branches/branch_3x/lucene/contrib/contrib-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/contrib-build.xml?rev=1135827&r1=1135826&r2=1135827&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/contrib-build.xml (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/contrib-build.xml Tue Jun 14 22:20:03 2011
@@ -76,6 +76,9 @@
</available>
<target name="dist-maven" if="pom.xml.present" depends="compile-core, jar-src">
+ <taskdef resource="org/apache/maven/artifact/ant/antlib.xml"
+ uri="antlib:org.apache.maven.artifact.ant"
+ classpathref="maven-ant-tasks.classpath"/>
<sequential>
<m2-deploy>
<artifact-attachments>
Modified: lucene/dev/branches/branch_3x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/CHANGES.txt?rev=1135827&r1=1135826&r2=1135827&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/solr/CHANGES.txt Tue Jun 14 22:20:03 2011
@@ -68,6 +68,15 @@ Other Changes
(James Dyer via rmuir)
* SOLR-2574: Upgrade slf4j to v1.6.1 (shalin)
+
+* LUCENE-3204: The maven-ant-tasks jar is now included in the source tree;
+ users of the generate-maven-artifacts target no longer have to manually
+ place this jar in the Ant classpath. NOTE: when Ant looks for the
+ maven-ant-tasks jar, it looks first in its pre-existing classpath, so
+ any copies it finds will be used instead of the copy included in the
+ Lucene/Solr source tree. For this reason, it is recommeded to remove
+ any copies of the maven-ant-tasks jar in the Ant classpath, e.g. under
+ ~/.ant/lib/ or under the Ant installation's lib/ directory. (Steve Rowe)
================== 3.2.0 ==================
Versions of Major Components
Modified: lucene/dev/branches/branch_3x/solr/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/build.xml?rev=1135827&r1=1135826&r2=1135827&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/build.xml (original)
+++ lucene/dev/branches/branch_3x/solr/build.xml Tue Jun 14 22:20:03 2011
@@ -995,7 +995,10 @@
<target name="stage" depends="prepare-release, copy-to-stage"/>
- <target name="generate-maven-artifacts" depends="maven.ant.tasks-check,dist,dist-src,dist-javadoc">
+ <target name="generate-maven-artifacts" depends="dist,dist-src,dist-javadoc">
+ <taskdef uri="antlib:org.apache.maven.artifact.ant"
+ resource="org/apache/maven/artifact/ant/antlib.xml"
+ classpathref="maven-ant-tasks.classpath" />
<sequential>
<ant target="get-maven-poms" dir=".."/>
Modified: lucene/dev/branches/branch_3x/solr/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/common-build.xml?rev=1135827&r1=1135826&r2=1135827&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/common-build.xml (original)
+++ lucene/dev/branches/branch_3x/solr/common-build.xml Tue Jun 14 22:20:03 2011
@@ -172,7 +172,11 @@
<property name="m2.repository.url" value="file://${maven.dist.dir}"/>
<property name="m2.repository.private.key" value="${user.home}/.ssh/id_dsa"/>
- <available property="maven.ant.tasks.present" classname="org.apache.maven.artifact.ant.Pom" />
+ <path id="maven-ant-tasks.classpath">
+ <fileset dir="${common-solr.dir}/../lucene/lib">
+ <include name="maven-ant-tasks-*.jar"/>
+ </fileset>
+ </path>
<!-- End Maven Properties -->
@@ -587,19 +591,6 @@
</manifest>
</target>
- <target name="maven.ant.tasks-check">
- <fail unless="maven.ant.tasks.present">#
- ##########################################################################
- Maven ant tasks not found.
-
- Please download the Maven ant tasks JAR (maven-ant-tasks-2.1.1.jar)
- from http://maven.apache.org/ant-tasks/download.html and add it to your
- $$HOME/.ant/lib/ directory, or to your $$ANT_HOME/lib/ directory, or
- to your $$CLASSPATH, or add "-lib /path/to/maven-ant-tasks-2.1.1.jar"
- to the ant command.
- ##########################################################################
- </fail>
- </target>
<!-- Validation -->
<target name="validate" depends="validate-solr"/>
<target name="validate-solr" depends="check-legal-solr" unless="validated-solr"/>