You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-user@lucene.apache.org by Ankit Murarka <an...@rancoretech.com> on 2013/09/01 13:28:22 UTC
Stream Closed Exception and Lock Obtain Failed Exception while reading
the file in chunks iteratively.
Hello.
I am struck in a problem and have been continously getting exception of
/*Stream Closed and LockObtainFailedException..*/
I am trying to read the complete document line by line and once I have
read 100000 lines I will add the doc to the writer and then again read
the next 100000 lines and repeat the same process unless the complete
file is read.
I cannot read the entire document line by line in 1 go due to memory
constraints.
Can anyone kindly let me know what might be wrong with this code:
The */StreamCLOSED/ *exception occurs when I try to add doc to writer
for iterations>1..In first iteration everything goes fine. /*Lock
exception */follows streamclosed exception..
Code Snippet:
package com.iorg;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.LongField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LiveIndexWriterConfig;
import org.apache.lucene.index.LogDocMergePolicy;
import org.apache.lucene.index.LogMergePolicy;
import org.apache.lucene.index.MergePolicy;
import org.apache.lucene.index.MergePolicy.OneMerge;
import org.apache.lucene.index.MergeScheduler;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
import com.rancore.CustomAnalyzerForCaseSensitive;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.util.Date;
import java.util.Iterator;
public class MainClass2 {
public static void main(String[] args) {
String indexPath = args[0]; //Place where indexes will be created
String docsPath=args[1]; //Place where the files are kept.
final File docDir = new File(docsPath);
if (!docDir.exists() || !docDir.canRead()) {
System.out.println("Document directory '"
+docDir.getAbsolutePath()+ "' does not exist or is not readable, please
check the path");
System.exit(1);
}
Date start = new Date();
try {
System.out.println("Indexing to directory ONLY '" + indexPath +
"'..."+docsPath);
Directory dir = FSDirectory.open(new File(indexPath));
Analyzer analyzer=new
CustomAnalyzerForCaseSensitive(Version.LUCENE_44);
IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_44,
analyzer);
iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);
if(args[2].trim().equalsIgnoreCase("OverAll")){
System.out.println("IN");
indexDocs(docDir,true,dir,iwc);
}
Date end = new Date();
System.out.println(end.getTime() - start.getTime() + " total
milliseconds");
} catch (IOException e) {
System.out.println(" caught a " + e.getClass() +
"\n with message: " + e.getMessage());
}
catch(Exception e)
{
e.printStackTrace();
}
}
//Over All
static void indexDocs(File file,boolean flag,Directory
dir,IndexWriterConfig iwc)
throws IOException {
FileInputStream fis = null;
if (file.canRead()) {
if (file.isDirectory())
{
String[] files = file.list();
System.out.println("size of list is " + files.length);
if (files != null) {
for (int i = 0; i < files.length; i++) {
System.out.println("Invoked for " + i + "and " +
files[i]);
indexDocs(new File(file, files[i]),flag,dir,iwc);
}
}
}
else {
boolean flags=true;
try {
fis = new FileInputStream(file);
} catch (FileNotFoundException fnfe) {
fnfe.printStackTrace();
}
try {
Document doc = new Document();
LineNumberReader lnr=new LineNumberReader(new FileReader(file));
Field pathField = new StringField("path", file.getPath(),
Field.Store.YES);
doc.add(pathField);
String line=null;
System.out.println("INITIALIZING");
int i=0;
doc.add(new StringField("TT",file.getName(),Field.Store.YES));
BufferedReader br=new BufferedReader(new
InputStreamReader(fis));
doc.add(new TextField("DD", br));
while(flags)
{
System.out.println("Looping");
IndexWriter iwcTemp1=new IndexWriter(dir,iwc);
while( null != (line = lnr.readLine()) ){
i++;
StringField sf=new
StringField("EEE",line.trim(),Field.Store.YES);
doc.add(sf);
if(i%10000==0)
{
System.out.println("Breaking" + i);
lnr.mark(i);
break;
}
sf=null;
}
if(line==null)
{
System.out.println("FALSE");
flags=false;
}
System.out.println("Total value is " + i);
if (iwcTemp1.getConfig().getOpenMode() ==
OpenMode.CREATE_OR_APPEND) {
try
{
iwcTemp1.addDocument(doc);
iwcTemp1.commit();
iwcTemp1.close();
}catch(Throwable t)
{
lnr.close();
br.close();
fis.close();
t.printStackTrace();
}
} else {
try
{
System.out.println("updating " + file);
iwcTemp1.updateDocument(new Term("path", file.getPath()), doc);
}catch(Exception e)
{
e.printStackTrace();
}
}
System.out.println("END OF WHILE ");
lnr.reset();
}//end of While
}catch (Exception e) {
e.printStackTrace();
}finally {
fis.close();
}
}
}
}
}
Kindly guide as to where the possible problem lies. Trying to figure out
but to no avail..
--
Regards
Ankit Murarka
"What lies behind us and what lies before us are tiny matters compared with what lies within us"
Re: Stream Closed Exception and Lock Obtain Failed Exception while
reading the file in chunks iteratively.
Posted by Erick Erickson <er...@gmail.com>.
bq: I think you might be referring to the close method not present in catch
method
This section of code will close iwcTemp1 in the "if" clause but not in the
"else" clause even if there are no exceptions thrown. And if there _are_
any exceptions thrown, as other's pointed out, then the iwcTemp1 _never_
gets closed. Trust Uwe when he tells you that this exception is caused by
not closing your IndexWriter.
if (iwcTemp1.getConfig().getOpenMode() ==
OpenMode.CREATE_OR_APPEND) {
try {
iwcTemp1.addDocument(doc);
iwcTemp1.commit();
iwcTemp1.close();
} catch (Throwable t) {
lnr.close();
br.close();
fis.close();
t.printStackTrace();
//***** iwcTemp1 never closed here
}
} else {
//***** iwcTemp1 never closed here
try {
System.out.println("updating " + file);
iwcTemp1.updateDocument(new Term("path",
file.getPath()), doc);
} catch (Exception e) {
e.printStackTrace();
}
}
You're piling work-arounds on top of incorrect code and getting further and
further down a wrong path. Correct the code first, _then_ think about
work-arounds.
Best,
Erick
On Mon, Sep 2, 2013 at 2:10 AM, Ankit Murarka <ankit.murarka@rancoretech.com
> wrote:
> There's a reason why Writer is being opened everytime inside a while loop.
> I usually open writer in main method itself as suggested by you and pass a
> reference to it. However what I have observed is that if my file contains
> more than 4 lakh lines, the writer.add(doc) line does not execute and
> throws an OutOfMemoryError although JVM is provided with enough heap.
>
> You may please go through my another query on mailing list for the same
> under the heading :
> "Files greater than 20 MB not getting Indexed. No files generated except
> write.lock even after 8-9 minutes. "
>
> So, as a precautionary measure, I will read 300000 lines from a document,
> add them to writer and then again read another 300000 lines and add them to
> writer with same filename and path... This process needs to be carried out
> recursively until the whole file is read.
>
> If there's another alternative I will be more than happy to know .
>
> As of now, I still get StreamClosedException and
> LockObtainFailedException. So any help on this will be deeply appreciated..
>
>
> On 9/1/2013 5:46 PM, Erick Erickson wrote:
>
>> I really recommend you restructure your program, it's a hard to follow.
>>
>> For instance, you open a new IndexWriter every time through the
>> while (flags)
>> loop. You only close it in the
>> if (iwcTemp1.getConfig().**getOpenMode() == OpenMode.CREATE_OR_APPEND) {
>> case. That may be the root of your problem right there.
>>
>> I'd probably open the indexWriter once in the main method, probably make
>> into a
>> member variable that's opened in an init method and closed in a finish
>> method
>> or some such.
>>
>> I'd separate the loop that traverses the directory structure away from the
>> code that indexes the data.
>>
>> FWIW,
>> Erick
>>
>>
>> On Sun, Sep 1, 2013 at 7:28 AM, Ankit Murarka<ankit.murarka@**
>> rancoretech.com <an...@rancoretech.com>
>>
>>
>>> wrote:
>>>
>>>
>>
>>
>>> Hello.
>>> I am struck in a problem and have been continously getting exception of
>>> /*Stream Closed and LockObtainFailedException..*/
>>>
>>> I am trying to read the complete document line by line and once I have
>>> read 100000 lines I will add the doc to the writer and then again read
>>> the
>>> next 100000 lines and repeat the same process unless the complete file is
>>> read.
>>>
>>> I cannot read the entire document line by line in 1 go due to memory
>>> constraints.
>>>
>>> Can anyone kindly let me know what might be wrong with this code:
>>>
>>> The */StreamCLOSED/ *exception occurs when I try to add doc to writer for
>>> iterations>1..In first iteration everything goes fine. /*Lock exception
>>> */follows streamclosed exception..
>>>
>>> Code Snippet:
>>>
>>> package com.iorg;
>>>
>>> import org.apache.lucene.analysis.****Analyzer;
>>> import org.apache.lucene.document.****Document;
>>> import org.apache.lucene.document.****Field;
>>> import org.apache.lucene.document.****LongField;
>>> import org.apache.lucene.document.****StringField;
>>> import org.apache.lucene.document.****TextField;
>>> import org.apache.lucene.index.****IndexCommit;
>>> import org.apache.lucene.index.****IndexWriter;
>>> import org.apache.lucene.index.****IndexWriterConfig.OpenMode;
>>> import org.apache.lucene.index.****IndexWriterConfig;
>>> import org.apache.lucene.index.****IndexableField;
>>> import org.apache.lucene.index.****LiveIndexWriterConfig;
>>> import org.apache.lucene.index.****LogDocMergePolicy;
>>> import org.apache.lucene.index.****LogMergePolicy;
>>> import org.apache.lucene.index.****MergePolicy;
>>> import org.apache.lucene.index.****MergePolicy.OneMerge;
>>> import org.apache.lucene.index.****MergeScheduler;
>>> import org.apache.lucene.index.Term;
>>> import org.apache.lucene.store.****Directory;
>>> import org.apache.lucene.store.****FSDirectory;
>>> import org.apache.lucene.util.****Version;
>>>
>>> import com.rancore.****CustomAnalyzerForCaseSensitive****;
>>>
>>>
>>> import java.io.BufferedReader;
>>> import java.io.File;
>>> import java.io.FileInputStream;
>>> import java.io.FileNotFoundException;
>>> import java.io.FileReader;
>>> import java.io.IOException;
>>> import java.io.InputStreamReader;
>>> import java.io.LineNumberReader;
>>> import java.util.Date;
>>> import java.util.Iterator;
>>>
>>> public class MainClass2 {
>>> public static void main(String[] args) {
>>>
>>> String indexPath = args[0]; //Place where indexes will be created
>>> String docsPath=args[1]; //Place where the files are kept.
>>>
>>> final File docDir = new File(docsPath);
>>> if (!docDir.exists() || !docDir.canRead()) {
>>> System.out.println("Document directory '"
>>> +docDir.getAbsolutePath()+
>>> "' does not exist or is not readable, please check the path");
>>> System.exit(1);
>>> }
>>> Date start = new Date();
>>> try {
>>> System.out.println("Indexing to directory ONLY '" + indexPath +
>>> "'..."+docsPath);
>>> Directory dir = FSDirectory.open(new File(indexPath));
>>> Analyzer analyzer=new CustomAnalyzerForCaseSensitive****
>>> (Version.LUCENE_44);
>>> IndexWriterConfig iwc = new IndexWriterConfig(Version.****
>>> LUCENE_44,
>>> analyzer);
>>> iwc.setOpenMode(OpenMode.****CREATE_OR_APPEND);
>>>
>>> if(args[2].trim().****equalsIgnoreCase("OverAll")){
>>> System.out.println("IN");
>>> indexDocs(docDir,true,dir,iwc)****;
>>>
>>> }
>>> Date end = new Date();
>>> System.out.println(end.****getTime() - start.getTime() + " total
>>>
>>> milliseconds");
>>>
>>> } catch (IOException e) {
>>> System.out.println(" caught a " + e.getClass() +
>>> "\n with message: " + e.getMessage());
>>> }
>>> catch(Exception e)
>>> {
>>> e.printStackTrace();
>>> }
>>> }
>>>
>>> //Over All
>>> static void indexDocs(File file,boolean flag,Directory
>>> dir,IndexWriterConfig iwc)
>>> throws IOException {
>>> FileInputStream fis = null;
>>> if (file.canRead()) {
>>> if (file.isDirectory())
>>> {
>>> String[] files = file.list();
>>> System.out.println("size of list is " + files.length);
>>> if (files != null) {
>>> for (int i = 0; i< files.length; i++) {
>>> System.out.println("Invoked for " + i + "and " +
>>> files[i]);
>>> indexDocs(new File(file, files[i]),flag,dir,iwc);
>>> }
>>> }
>>> }
>>> else {
>>> boolean flags=true;
>>> try {
>>> fis = new FileInputStream(file);
>>> } catch (FileNotFoundException fnfe) {
>>> fnfe.printStackTrace();
>>> }
>>> try {
>>> Document doc = new Document();
>>> LineNumberReader lnr=new LineNumberReader(new
>>> FileReader(file));
>>> Field pathField = new StringField("path", file.getPath(),
>>> Field.Store.YES);
>>> doc.add(pathField);
>>> String line=null;
>>> System.out.println("****INITIALIZING");
>>> int i=0;
>>> doc.add(new StringField("TT",file.getName(**
>>> **),Field.Store.YES));
>>>
>>> BufferedReader br=new BufferedReader(new
>>> InputStreamReader(fis));
>>> doc.add(new TextField("DD", br));
>>> while(flags)
>>> {
>>> System.out.println("Looping");
>>> IndexWriter iwcTemp1=new IndexWriter(dir,iwc);
>>> while( null != (line = lnr.readLine()) ){
>>> i++;
>>> StringField sf=new StringField("EEE",line.trim(),****
>>>
>>> Field.Store.YES);
>>> doc.add(sf);
>>> if(i%10000==0)
>>> {
>>> System.out.println("Breaking" + i);
>>> lnr.mark(i);
>>> break;
>>> }
>>> sf=null;
>>> }
>>> if(line==null)
>>> {
>>> System.out.println("FALSE");
>>> flags=false;
>>> }
>>> System.out.println("Total value is " + i);
>>> if (iwcTemp1.getConfig().****getOpenMode() ==
>>>
>>> OpenMode.CREATE_OR_APPEND) {
>>> try
>>> {
>>> iwcTemp1.addDocument(doc);
>>> iwcTemp1.commit();
>>> iwcTemp1.close();
>>> }catch(Throwable t)
>>> {
>>> lnr.close();
>>> br.close();
>>> fis.close();
>>> t.printStackTrace();
>>> }
>>>
>>> } else {
>>> try
>>> {
>>>
>>> System.out.println("updating " + file);
>>> iwcTemp1.updateDocument(new Term("path", file.getPath()),
>>> doc);
>>> }catch(Exception e)
>>> {
>>> e.printStackTrace();
>>> }
>>> }
>>> System.out.println("END OF WHILE ");
>>> lnr.reset();
>>> }//end of While
>>> }catch (Exception e) {
>>> e.printStackTrace();
>>> }finally {
>>> fis.close();
>>> }
>>> }
>>> }
>>> }
>>> }
>>>
>>>
>>> Kindly guide as to where the possible problem lies. Trying to figure out
>>> but to no avail..
>>>
>>> --
>>> Regards
>>>
>>> Ankit Murarka
>>>
>>> "What lies behind us and what lies before us are tiny matters compared
>>> with what lies within us"
>>>
>>>
>>>
>>>
>>
>>
>
>
> --
> Regards
>
> Ankit Murarka
>
> "What lies behind us and what lies before us are tiny matters compared
> with what lies within us"
>
>
> ------------------------------**------------------------------**---------
> To unsubscribe, e-mail: java-user-unsubscribe@lucene.**apache.org<ja...@lucene.apache.org>
> For additional commands, e-mail: java-user-help@lucene.apache.**org<ja...@lucene.apache.org>
>
>
RE: Stream Closed Exception and Lock Obtain Failed Exception while reading the file in chunks iteratively.
Posted by Uwe Schindler <uw...@thetaphi.de>.
Hi,
LockObtainFailed is *always* caused by a missing close of the IndexWriter. The code is ununderstandable and things like the evil Eclipse "automated-printStackTrace-catch-block" make it obvious that the code is not designed correctly. To write good code, please *disable* this automatic feature of Eclipse it is the source of *all* those errors. Declare all thrown Exceptions on the methods and do the handling correctly without suppressing them on the level of opening/writing IndexWriter using catch and finally blocks.
The cause of the OOM when not reopening IW can be caused by incorrect Exception handling, too. In addition, instead of opening/closing IndexWriter you can call commit().
It would be good to see the code that writes to the index without reopening the reader all the time, to find the root cause of your problem you are trying to workaround.
Uwe
-----
Uwe Schindler
H.-H.-Meier-Allee 63, D-28213 Bremen
http://www.thetaphi.de
eMail: uwe@thetaphi.de
> -----Original Message-----
> From: Ankit Murarka [mailto:ankit.murarka@rancoretech.com]
> Sent: Monday, September 02, 2013 8:57 AM
> To: java-user@lucene.apache.org
> Subject: Re: Stream Closed Exception and Lock Obtain Failed Exception while
> reading the file in chunks iteratively.
>
> Hello..
>
> I think you might be referring to the close method not present in catch
> method. if that's so, that is purposely done for the time being...
>
> Else if the execution happens properly, then the flow won't go to catch as of
> now. Obviously the close will be present both in catch and finally block.
>
> But I don't think this is possibly the root of this problem.
>
> There seems to be another problem which is causing this issue..Would
> appreciate some guidance..
>
> On 9/2/2013 12:11 PM, Trejkaz wrote:
> > On Mon, Sep 2, 2013 at 4:10 PM, Ankit Murarka
> > <an...@rancoretech.com> wrote:
> >
> >> There's a reason why Writer is being opened everytime inside a while
> >> loop. I usually open writer in main method itself as suggested by you
> >> and pass a reference to it. However what I have observed is that if
> >> my file contains more than 4 lakh lines, the writer.add(doc) line
> >> does not execute and throws an OutOfMemoryError although JVM is
> provided with enough heap.
> >>
> > The problem isn't opening it multiple times (although that does seem
> > unnecessary too), the problem is that half the time you're not closing
> > it.
> >
> > Try refactoring to always use the try(){} syntax so that all your
> > Closeable objects are being closed properly. You'll find all sorts of
> > issues with the code you posted.
> >
> > TX
> >
> > ---------------------------------------------------------------------
> > To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
> > For additional commands, e-mail: java-user-help@lucene.apache.org
> >
> >
> >
>
>
> --
> Regards
>
> Ankit Murarka
>
> "What lies behind us and what lies before us are tiny matters compared with
> what lies within us"
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
> For additional commands, e-mail: java-user-help@lucene.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
For additional commands, e-mail: java-user-help@lucene.apache.org
Re: Stream Closed Exception and Lock Obtain Failed Exception while
reading the file in chunks iteratively.
Posted by Ankit Murarka <an...@rancoretech.com>.
Hello..
I think you might be referring to the close method not present in catch
method. if that's so, that is purposely done for the time being...
Else if the execution happens properly, then the flow won't go to catch
as of now. Obviously the close will be present both in catch and finally
block.
But I don't think this is possibly the root of this problem.
There seems to be another problem which is causing this issue..Would
appreciate some guidance..
On 9/2/2013 12:11 PM, Trejkaz wrote:
> On Mon, Sep 2, 2013 at 4:10 PM, Ankit Murarka
> <an...@rancoretech.com> wrote:
>
>> There's a reason why Writer is being opened everytime inside a while loop. I
>> usually open writer in main method itself as suggested by you and pass a
>> reference to it. However what I have observed is that if my file contains
>> more than 4 lakh lines, the writer.add(doc) line does not execute and throws
>> an OutOfMemoryError although JVM is provided with enough heap.
>>
> The problem isn't opening it multiple times (although that does seem
> unnecessary too), the problem is that half the time you're not closing
> it.
>
> Try refactoring to always use the try(){} syntax so that all your
> Closeable objects are being closed properly. You'll find all sorts of
> issues with the code you posted.
>
> TX
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
> For additional commands, e-mail: java-user-help@lucene.apache.org
>
>
>
--
Regards
Ankit Murarka
"What lies behind us and what lies before us are tiny matters compared with what lies within us"
---------------------------------------------------------------------
To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
For additional commands, e-mail: java-user-help@lucene.apache.org
Re: Stream Closed Exception and Lock Obtain Failed Exception while
reading the file in chunks iteratively.
Posted by Trejkaz <tr...@trypticon.org>.
On Mon, Sep 2, 2013 at 4:10 PM, Ankit Murarka
<an...@rancoretech.com> wrote:
> There's a reason why Writer is being opened everytime inside a while loop. I
> usually open writer in main method itself as suggested by you and pass a
> reference to it. However what I have observed is that if my file contains
> more than 4 lakh lines, the writer.add(doc) line does not execute and throws
> an OutOfMemoryError although JVM is provided with enough heap.
The problem isn't opening it multiple times (although that does seem
unnecessary too), the problem is that half the time you're not closing
it.
Try refactoring to always use the try(){} syntax so that all your
Closeable objects are being closed properly. You'll find all sorts of
issues with the code you posted.
TX
---------------------------------------------------------------------
To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
For additional commands, e-mail: java-user-help@lucene.apache.org
Re: Stream Closed Exception and Lock Obtain Failed Exception while
reading the file in chunks iteratively.
Posted by Ankit Murarka <an...@rancoretech.com>.
There's a reason why Writer is being opened everytime inside a while
loop. I usually open writer in main method itself as suggested by you
and pass a reference to it. However what I have observed is that if my
file contains more than 4 lakh lines, the writer.add(doc) line does not
execute and throws an OutOfMemoryError although JVM is provided with
enough heap.
You may please go through my another query on mailing list for the same
under the heading :
"Files greater than 20 MB not getting Indexed. No files generated except
write.lock even after 8-9 minutes. "
So, as a precautionary measure, I will read 300000 lines from a
document, add them to writer and then again read another 300000 lines
and add them to writer with same filename and path... This process needs
to be carried out recursively until the whole file is read.
If there's another alternative I will be more than happy to know .
As of now, I still get StreamClosedException and
LockObtainFailedException. So any help on this will be deeply appreciated..
On 9/1/2013 5:46 PM, Erick Erickson wrote:
> I really recommend you restructure your program, it's a hard to follow.
>
> For instance, you open a new IndexWriter every time through the
> while (flags)
> loop. You only close it in the
> if (iwcTemp1.getConfig().getOpenMode() == OpenMode.CREATE_OR_APPEND) {
> case. That may be the root of your problem right there.
>
> I'd probably open the indexWriter once in the main method, probably make
> into a
> member variable that's opened in an init method and closed in a finish
> method
> or some such.
>
> I'd separate the loop that traverses the directory structure away from the
> code that indexes the data.
>
> FWIW,
> Erick
>
>
> On Sun, Sep 1, 2013 at 7:28 AM, Ankit Murarka<ankit.murarka@rancoretech.com
>
>> wrote:
>>
>
>> Hello.
>> I am struck in a problem and have been continously getting exception of
>> /*Stream Closed and LockObtainFailedException..*/
>>
>> I am trying to read the complete document line by line and once I have
>> read 100000 lines I will add the doc to the writer and then again read the
>> next 100000 lines and repeat the same process unless the complete file is
>> read.
>>
>> I cannot read the entire document line by line in 1 go due to memory
>> constraints.
>>
>> Can anyone kindly let me know what might be wrong with this code:
>>
>> The */StreamCLOSED/ *exception occurs when I try to add doc to writer for
>> iterations>1..In first iteration everything goes fine. /*Lock exception
>> */follows streamclosed exception..
>>
>> Code Snippet:
>>
>> package com.iorg;
>>
>> import org.apache.lucene.analysis.**Analyzer;
>> import org.apache.lucene.document.**Document;
>> import org.apache.lucene.document.**Field;
>> import org.apache.lucene.document.**LongField;
>> import org.apache.lucene.document.**StringField;
>> import org.apache.lucene.document.**TextField;
>> import org.apache.lucene.index.**IndexCommit;
>> import org.apache.lucene.index.**IndexWriter;
>> import org.apache.lucene.index.**IndexWriterConfig.OpenMode;
>> import org.apache.lucene.index.**IndexWriterConfig;
>> import org.apache.lucene.index.**IndexableField;
>> import org.apache.lucene.index.**LiveIndexWriterConfig;
>> import org.apache.lucene.index.**LogDocMergePolicy;
>> import org.apache.lucene.index.**LogMergePolicy;
>> import org.apache.lucene.index.**MergePolicy;
>> import org.apache.lucene.index.**MergePolicy.OneMerge;
>> import org.apache.lucene.index.**MergeScheduler;
>> import org.apache.lucene.index.Term;
>> import org.apache.lucene.store.**Directory;
>> import org.apache.lucene.store.**FSDirectory;
>> import org.apache.lucene.util.**Version;
>>
>> import com.rancore.**CustomAnalyzerForCaseSensitive**;
>>
>> import java.io.BufferedReader;
>> import java.io.File;
>> import java.io.FileInputStream;
>> import java.io.FileNotFoundException;
>> import java.io.FileReader;
>> import java.io.IOException;
>> import java.io.InputStreamReader;
>> import java.io.LineNumberReader;
>> import java.util.Date;
>> import java.util.Iterator;
>>
>> public class MainClass2 {
>> public static void main(String[] args) {
>>
>> String indexPath = args[0]; //Place where indexes will be created
>> String docsPath=args[1]; //Place where the files are kept.
>>
>> final File docDir = new File(docsPath);
>> if (!docDir.exists() || !docDir.canRead()) {
>> System.out.println("Document directory '" +docDir.getAbsolutePath()+
>> "' does not exist or is not readable, please check the path");
>> System.exit(1);
>> }
>> Date start = new Date();
>> try {
>> System.out.println("Indexing to directory ONLY '" + indexPath +
>> "'..."+docsPath);
>> Directory dir = FSDirectory.open(new File(indexPath));
>> Analyzer analyzer=new CustomAnalyzerForCaseSensitive**
>> (Version.LUCENE_44);
>> IndexWriterConfig iwc = new IndexWriterConfig(Version.**LUCENE_44,
>> analyzer);
>> iwc.setOpenMode(OpenMode.**CREATE_OR_APPEND);
>>
>> if(args[2].trim().**equalsIgnoreCase("OverAll")){
>> System.out.println("IN");
>> indexDocs(docDir,true,dir,iwc)**;
>> }
>> Date end = new Date();
>> System.out.println(end.**getTime() - start.getTime() + " total
>> milliseconds");
>>
>> } catch (IOException e) {
>> System.out.println(" caught a " + e.getClass() +
>> "\n with message: " + e.getMessage());
>> }
>> catch(Exception e)
>> {
>> e.printStackTrace();
>> }
>> }
>>
>> //Over All
>> static void indexDocs(File file,boolean flag,Directory
>> dir,IndexWriterConfig iwc)
>> throws IOException {
>> FileInputStream fis = null;
>> if (file.canRead()) {
>> if (file.isDirectory())
>> {
>> String[] files = file.list();
>> System.out.println("size of list is " + files.length);
>> if (files != null) {
>> for (int i = 0; i< files.length; i++) {
>> System.out.println("Invoked for " + i + "and " +
>> files[i]);
>> indexDocs(new File(file, files[i]),flag,dir,iwc);
>> }
>> }
>> }
>> else {
>> boolean flags=true;
>> try {
>> fis = new FileInputStream(file);
>> } catch (FileNotFoundException fnfe) {
>> fnfe.printStackTrace();
>> }
>> try {
>> Document doc = new Document();
>> LineNumberReader lnr=new LineNumberReader(new FileReader(file));
>> Field pathField = new StringField("path", file.getPath(),
>> Field.Store.YES);
>> doc.add(pathField);
>> String line=null;
>> System.out.println("**INITIALIZING");
>> int i=0;
>> doc.add(new StringField("TT",file.getName(**),Field.Store.YES));
>> BufferedReader br=new BufferedReader(new InputStreamReader(fis));
>> doc.add(new TextField("DD", br));
>> while(flags)
>> {
>> System.out.println("Looping");
>> IndexWriter iwcTemp1=new IndexWriter(dir,iwc);
>> while( null != (line = lnr.readLine()) ){
>> i++;
>> StringField sf=new StringField("EEE",line.trim(),**
>> Field.Store.YES);
>> doc.add(sf);
>> if(i%10000==0)
>> {
>> System.out.println("Breaking" + i);
>> lnr.mark(i);
>> break;
>> }
>> sf=null;
>> }
>> if(line==null)
>> {
>> System.out.println("FALSE");
>> flags=false;
>> }
>> System.out.println("Total value is " + i);
>> if (iwcTemp1.getConfig().**getOpenMode() ==
>> OpenMode.CREATE_OR_APPEND) {
>> try
>> {
>> iwcTemp1.addDocument(doc);
>> iwcTemp1.commit();
>> iwcTemp1.close();
>> }catch(Throwable t)
>> {
>> lnr.close();
>> br.close();
>> fis.close();
>> t.printStackTrace();
>> }
>>
>> } else {
>> try
>> {
>>
>> System.out.println("updating " + file);
>> iwcTemp1.updateDocument(new Term("path", file.getPath()), doc);
>> }catch(Exception e)
>> {
>> e.printStackTrace();
>> }
>> }
>> System.out.println("END OF WHILE ");
>> lnr.reset();
>> }//end of While
>> }catch (Exception e) {
>> e.printStackTrace();
>> }finally {
>> fis.close();
>> }
>> }
>> }
>> }
>> }
>>
>>
>> Kindly guide as to where the possible problem lies. Trying to figure out
>> but to no avail..
>>
>> --
>> Regards
>>
>> Ankit Murarka
>>
>> "What lies behind us and what lies before us are tiny matters compared
>> with what lies within us"
>>
>>
>>
>
--
Regards
Ankit Murarka
"What lies behind us and what lies before us are tiny matters compared with what lies within us"
---------------------------------------------------------------------
To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
For additional commands, e-mail: java-user-help@lucene.apache.org
Re: Stream Closed Exception and Lock Obtain Failed Exception while
reading the file in chunks iteratively.
Posted by Erick Erickson <er...@gmail.com>.
I really recommend you restructure your program, it's a hard to follow.
For instance, you open a new IndexWriter every time through the
while (flags)
loop. You only close it in the
if (iwcTemp1.getConfig().getOpenMode() == OpenMode.CREATE_OR_APPEND) {
case. That may be the root of your problem right there.
I'd probably open the indexWriter once in the main method, probably make
into a
member variable that's opened in an init method and closed in a finish
method
or some such.
I'd separate the loop that traverses the directory structure away from the
code that indexes the data.
FWIW,
Erick
On Sun, Sep 1, 2013 at 7:28 AM, Ankit Murarka <ankit.murarka@rancoretech.com
> wrote:
> Hello.
> I am struck in a problem and have been continously getting exception of
> /*Stream Closed and LockObtainFailedException..*/
>
> I am trying to read the complete document line by line and once I have
> read 100000 lines I will add the doc to the writer and then again read the
> next 100000 lines and repeat the same process unless the complete file is
> read.
>
> I cannot read the entire document line by line in 1 go due to memory
> constraints.
>
> Can anyone kindly let me know what might be wrong with this code:
>
> The */StreamCLOSED/ *exception occurs when I try to add doc to writer for
> iterations>1..In first iteration everything goes fine. /*Lock exception
> */follows streamclosed exception..
>
> Code Snippet:
>
> package com.iorg;
>
> import org.apache.lucene.analysis.**Analyzer;
> import org.apache.lucene.document.**Document;
> import org.apache.lucene.document.**Field;
> import org.apache.lucene.document.**LongField;
> import org.apache.lucene.document.**StringField;
> import org.apache.lucene.document.**TextField;
> import org.apache.lucene.index.**IndexCommit;
> import org.apache.lucene.index.**IndexWriter;
> import org.apache.lucene.index.**IndexWriterConfig.OpenMode;
> import org.apache.lucene.index.**IndexWriterConfig;
> import org.apache.lucene.index.**IndexableField;
> import org.apache.lucene.index.**LiveIndexWriterConfig;
> import org.apache.lucene.index.**LogDocMergePolicy;
> import org.apache.lucene.index.**LogMergePolicy;
> import org.apache.lucene.index.**MergePolicy;
> import org.apache.lucene.index.**MergePolicy.OneMerge;
> import org.apache.lucene.index.**MergeScheduler;
> import org.apache.lucene.index.Term;
> import org.apache.lucene.store.**Directory;
> import org.apache.lucene.store.**FSDirectory;
> import org.apache.lucene.util.**Version;
>
> import com.rancore.**CustomAnalyzerForCaseSensitive**;
>
> import java.io.BufferedReader;
> import java.io.File;
> import java.io.FileInputStream;
> import java.io.FileNotFoundException;
> import java.io.FileReader;
> import java.io.IOException;
> import java.io.InputStreamReader;
> import java.io.LineNumberReader;
> import java.util.Date;
> import java.util.Iterator;
>
> public class MainClass2 {
> public static void main(String[] args) {
>
> String indexPath = args[0]; //Place where indexes will be created
> String docsPath=args[1]; //Place where the files are kept.
>
> final File docDir = new File(docsPath);
> if (!docDir.exists() || !docDir.canRead()) {
> System.out.println("Document directory '" +docDir.getAbsolutePath()+
> "' does not exist or is not readable, please check the path");
> System.exit(1);
> }
> Date start = new Date();
> try {
> System.out.println("Indexing to directory ONLY '" + indexPath +
> "'..."+docsPath);
> Directory dir = FSDirectory.open(new File(indexPath));
> Analyzer analyzer=new CustomAnalyzerForCaseSensitive**
> (Version.LUCENE_44);
> IndexWriterConfig iwc = new IndexWriterConfig(Version.**LUCENE_44,
> analyzer);
> iwc.setOpenMode(OpenMode.**CREATE_OR_APPEND);
>
> if(args[2].trim().**equalsIgnoreCase("OverAll")){
> System.out.println("IN");
> indexDocs(docDir,true,dir,iwc)**;
> }
> Date end = new Date();
> System.out.println(end.**getTime() - start.getTime() + " total
> milliseconds");
>
> } catch (IOException e) {
> System.out.println(" caught a " + e.getClass() +
> "\n with message: " + e.getMessage());
> }
> catch(Exception e)
> {
> e.printStackTrace();
> }
> }
>
> //Over All
> static void indexDocs(File file,boolean flag,Directory
> dir,IndexWriterConfig iwc)
> throws IOException {
> FileInputStream fis = null;
> if (file.canRead()) {
> if (file.isDirectory())
> {
> String[] files = file.list();
> System.out.println("size of list is " + files.length);
> if (files != null) {
> for (int i = 0; i < files.length; i++) {
> System.out.println("Invoked for " + i + "and " +
> files[i]);
> indexDocs(new File(file, files[i]),flag,dir,iwc);
> }
> }
> }
> else {
> boolean flags=true;
> try {
> fis = new FileInputStream(file);
> } catch (FileNotFoundException fnfe) {
> fnfe.printStackTrace();
> }
> try {
> Document doc = new Document();
> LineNumberReader lnr=new LineNumberReader(new FileReader(file));
> Field pathField = new StringField("path", file.getPath(),
> Field.Store.YES);
> doc.add(pathField);
> String line=null;
> System.out.println("**INITIALIZING");
> int i=0;
> doc.add(new StringField("TT",file.getName(**),Field.Store.YES));
> BufferedReader br=new BufferedReader(new InputStreamReader(fis));
> doc.add(new TextField("DD", br));
> while(flags)
> {
> System.out.println("Looping");
> IndexWriter iwcTemp1=new IndexWriter(dir,iwc);
> while( null != (line = lnr.readLine()) ){
> i++;
> StringField sf=new StringField("EEE",line.trim(),**
> Field.Store.YES);
> doc.add(sf);
> if(i%10000==0)
> {
> System.out.println("Breaking" + i);
> lnr.mark(i);
> break;
> }
> sf=null;
> }
> if(line==null)
> {
> System.out.println("FALSE");
> flags=false;
> }
> System.out.println("Total value is " + i);
> if (iwcTemp1.getConfig().**getOpenMode() ==
> OpenMode.CREATE_OR_APPEND) {
> try
> {
> iwcTemp1.addDocument(doc);
> iwcTemp1.commit();
> iwcTemp1.close();
> }catch(Throwable t)
> {
> lnr.close();
> br.close();
> fis.close();
> t.printStackTrace();
> }
>
> } else {
> try
> {
>
> System.out.println("updating " + file);
> iwcTemp1.updateDocument(new Term("path", file.getPath()), doc);
> }catch(Exception e)
> {
> e.printStackTrace();
> }
> }
> System.out.println("END OF WHILE ");
> lnr.reset();
> }//end of While
> }catch (Exception e) {
> e.printStackTrace();
> }finally {
> fis.close();
> }
> }
> }
> }
> }
>
>
> Kindly guide as to where the possible problem lies. Trying to figure out
> but to no avail..
>
> --
> Regards
>
> Ankit Murarka
>
> "What lies behind us and what lies before us are tiny matters compared
> with what lies within us"
>
>