You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-user@lucene.apache.org by Ridzwan Aminuddin <wa...@inbox.com> on 2007/02/22 09:20:12 UTC
Lucene 1.4.3 : IndexWriter.addDocument(doc) fails when run on OS
requiring permissions
Hi!
I'm writing a java program that uses Lucene 1.4.3 to index and create a vector file of words found in Text Files. The purpose is for text mining.
I created a Java .Jar file from my program and my python script calls the Java Jar executable. This is all triggered by my DTML code.
I'm running on Linux and i have no problem executing the script when i execute via command line. But once i trigger the script via the web (using Zope/Plone external methods ) it doesn't work anymore. This is because of the strict permissions that LInux has over its files and folders.
I've narrowed down the problem to the IndexWriter.addDocument(doc) method in Lucene 1.4.3 and as you can see below my code fails specifically when a new FieldsWriter object is being initialised.
I strongly suspect that it fails at this point but have no idea how to overcome this problem. I know that it has to do with the permissions because th eprogram works like a miracle when it is called via command line by the super user (sudo).
Could anyone give me any pointers or ideas of how i could overcome this.
The final statement which is printed before the program hangs is:
"Entering DocumentWriter.AddDocument.... (4)"
Here is the portions of my relevant code :
//-------------------------------------------------------------------------------------------
// Indexer.Java // This is my own method and class
//-------------------------------------------------------------------------------------------
// continued from some other code......
Document doc = new Document();
doc.add(Field.Text("articleTitle", articleTitle, true));
doc.add(Field.Text("articleURL", articleURL, true));
doc.add(Field.Text("articleSummary", articleSummary, true));
doc.add(Field.Text("articleDate", articleDate, true));
doc.add(Field.Text("articleSource", articleSource, true));
doc.add(Field.Text("articleBody", articleBody, true));
doc.add(Field.Keyword("filename", f.getCanonicalPath()));
try
{
writer.addDocument(doc); // indexing fails because this statement cannot be executed
}
catch (Exception e)
{
System.err.println ("Cannot add doc exception thrown!");
}
//-------------------------------------------------------------------------------------------
// IndexWriter.Java // Lucene 1.4.3
//-------------------------------------------------------------------------------------------
public void addDocument(Document doc) throws IOException {
addDocument(doc, analyzer);
}
public void addDocument(Document doc, Analyzer analyzer) throws IOException {
DocumentWriter dw;
dw = new DocumentWriter(ramDirectory, analyzer, similarity, maxFieldLength);
String segmentName = newSegmentName();
dw.addDocument(segmentName, doc); // The program fails to execute this line onwards!
synchronized (this) {
segmentInfos.addElement(new SegmentInfo(segmentName, 1, ramDirectory));
maybeMergeSegments();
}
}
//-------------------------------------------------------------------------------------------
// DocumentWriter.Java // Lucene 1.4.3
//-------------------------------------------------------------------------------------------
final void addDocument(String segment, Document doc)
throws IOException {
System.out.println("Entering DocumentWriter.AddDocument.... (1)");
// write field names
fieldInfos = new FieldInfos();
System.out.println("Entering DocumentWriter.AddDocument.... (2)");
fieldInfos.add(doc);
System.out.println("Entering DocumentWriter.AddDocument.... (3)");
fieldInfos.write(directory, segment + ".fnm");
System.out.println("Entering DocumentWriter.AddDocument.... (4)"); // The program fails after this
// write field values
FieldsWriter fieldsWriter =
new FieldsWriter(directory, segment, fieldInfos); // Program fails to execute this statement
System.out.println("Entering DocumentWriter.AddDocument.... (5)");
try {
fieldsWriter.addDocument(doc);
System.out.println("Entering DocumentWriter.AddDocument.... (6)");
} finally {
fieldsWriter.close();
System.out.println("Entering DocumentWriter.AddDocument.... (7)");
}
System.out.println("Entering DocumentWriter.AddDocument.... (8)");
// invert doc into postingTable
postingTable.clear(); // clear postingTable
fieldLengths = new int[fieldInfos.size()]; // init fieldLengths
fieldPositions = new int[fieldInfos.size()]; // init fieldPositions
System.out.println("Entering DocumentWriter.AddDocument.... (9)");
fieldBoosts = new float[fieldInfos.size()]; // init fieldBoosts
Arrays.fill(fieldBoosts, doc.getBoost());
System.out.println("Entering DocumentWriter.AddDocument.... (10)");
invertDocument(doc);
System.out.println("Entering DocumentWriter.AddDocument.... (11)");
// sort postingTable into an array
Posting[] postings = sortPostingTable();
System.out.println("Entering DocumentWriter.AddDocument.... (12)");
/*
for (int i = 0; i < postings.length; i++) {
Posting posting = postings[i];
System.out.print(posting.term);
System.out.print(" freq=" + posting.freq);
System.out.print(" pos=");
System.out.print(posting.positions[0]);
for (int j = 1; j < posting.freq; j++)
System.out.print("," + posting.positions[j]);
System.out.println("");
}
*/
// write postings
writePostings(postings, segment);
System.out.println("Entering DocumentWriter.AddDocument.... (13)");
// write norms of indexed fields
writeNorms(doc, segment);
System.out.println("Entering DocumentWriter.AddDocument.... (14)");
}
//-------------------------------------------------------------------------------------------
// FieldsWriter.Java // Lucene 1.4.3
//-------------------------------------------------------------------------------------------
FieldsWriter(Directory d, String segment, FieldInfos fn)
throws IOException {
fieldInfos = fn;
fieldsStream = d.createFile(segment + ".fdt");
indexStream = d.createFile(segment + ".fdx");
}
//-------------------------------------------------------------------------------------------
// FSDirectory.Java // Lucene 1.4.3
//-------------------------------------------------------------------------------------------
public final OutputStream createFile(String name) throws IOException {
System.out.println("Entering FSDirectory.createFile.... returning an OutputStream");
return new FSOutputStream(new File(directory, name));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
For additional commands, e-mail: java-user-help@lucene.apache.org
RE: Lucene 1.4.3 : IndexWriter.addDocument(doc) fails when run on OS requiring permissions
Posted by Steven Parkes <st...@esseff.org>.
The easiest way to pin this down is to get the backtrace from the
exception, e.g., e.printStackTrace(). That would tell a lot.
That said, prior to 2.1, lucene would put lock files outside the index
directory. I don't know if that's what you're hitting, though, because I
think the writer should have taken the lock when you created it, not
when you tried to do the addDocument.
-----Original Message-----
From: Ridzwan Aminuddin [mailto:wanster@inbox.com]
Sent: Thursday, February 22, 2007 8:48 PM
To: java-user@lucene.apache.org
Subject: Re: Lucene 1.4.3 : IndexWriter.addDocument(doc) fails when run
on OS requiring permissions
Hi Guys.
Ok thanks for the replies. You guys are right that it is to do with the
system and not with Lucene. However, what i'm trying to do is to
pinpoint and narrow down to the exact place that causes the system to
fail. and then from there try to remedy the problem.
The odd thing is that the program is still able to write other files to
the subdirectories that the program itself creates. It is only when it
goes through this indexing process that this program halts due to the
insufficient permissions. But the directory i provided to store the
target index files has been set to read/write/execute (drwxrwxrwx) all
permissions.
In any case, i suspect that it is due to this portion of code.
: FieldsWriter(Directory d, String segment, FieldInfos fn)
: throws IOException {
: fieldInfos = fn;
: fieldsStream = d.createFile(segment + ".fdt");
: indexStream = d.createFile(segment + ".fdx");
: }
Are these two files created in the Directory d?
And is this Directory d the ramDirectory that i have provided when i
called the method: dw = new DocumentWriter(ramDirectory, analyzer,
similarity,
maxFieldLength);
in IndexWriter.Java?
If yes. then where exactly does this ramDirectory point to? cos as far
as i can see from the code, ramDirectory is initialised as
RamDirectory ramDirectory = new RamDirectory()
I think the program fails when it tries to create these two files
somehow.
Please help to enlighten me?... Maybe knowing the exact path to this
ramDirectory would help me in finding out which folder i need to provide
access to....
Also, does Lucene ever write any temp data to /tmp ?
> -----Original Message-----
> From: hossman_lucene@fucit.org
> Sent: Thu, 22 Feb 2007 11:43:38 -0800 (PST)
> To: java-user@lucene.apache.org
> Subject: Re: Lucene 1.4.3 : IndexWriter.addDocument(doc) fails when
run
> on OS requiring permissions
>
>
> This sounds like it has absolutely nothing to do with Lucene, and
> everything to do with good security permissions -- your Zope/python
front
> end is most likely running as a user thta does not have write
permissions
> to the directory where your index lives. you'll need to remedy that.
>
> you can write a simple java app that doens't use lucene at all -- just
> creates a file and writes "hellow world" to it -- and you will most
> likely see this exact same behavior, dealing with teh file permissions
is
> totally out side the scope of Lucene.
>
>
> : Date: Thu, 22 Feb 2007 00:20:12 -0800
> : From: Ridzwan Aminuddin <wa...@inbox.com>
> : Reply-To: java-user@lucene.apache.org
> : To: java-user@lucene.apache.org
> : Subject: Lucene 1.4.3 : IndexWriter.addDocument(doc) fails when run
on
> OS
> : requiring permissions
> :
> : Hi!
> :
> : I'm writing a java program that uses Lucene 1.4.3 to index and
create a
> vector file of words found in Text Files. The purpose is for text
mining.
> :
> : I created a Java .Jar file from my program and my python script
calls
> the Java Jar executable. This is all triggered by my DTML code.
> :
> : I'm running on Linux and i have no problem executing the script when
i
> execute via command line. But once i trigger the script via the web
> (using Zope/Plone external methods ) it doesn't work anymore. This is
> because of the strict permissions that LInux has over its files and
> folders.
> :
> : I've narrowed down the problem to the IndexWriter.addDocument(doc)
> method in Lucene 1.4.3 and as you can see below my code fails
> specifically when a new FieldsWriter object is being initialised.
> :
> : I strongly suspect that it fails at this point but have no idea how
to
> overcome this problem. I know that it has to do with the permissions
> because th eprogram works like a miracle when it is called via command
> line by the super user (sudo).
> :
> : Could anyone give me any pointers or ideas of how i could overcome
> this.
> :
> : The final statement which is printed before the program hangs is:
> : "Entering DocumentWriter.AddDocument.... (4)"
> :
> : Here is the portions of my relevant code :
> :
> :
> :
> :
>
//----------------------------------------------------------------------
---------------------
> : // Indexer.Java // This is my own method and class
> :
>
//----------------------------------------------------------------------
---------------------
> : // continued from some other code......
> :
> : Document doc = new Document();
> :
> : doc.add(Field.Text("articleTitle", articleTitle, true));
> : doc.add(Field.Text("articleURL", articleURL, true));
> : doc.add(Field.Text("articleSummary", articleSummary, true));
> : doc.add(Field.Text("articleDate", articleDate, true));
> : doc.add(Field.Text("articleSource", articleSource, true));
> : doc.add(Field.Text("articleBody", articleBody, true));
> : doc.add(Field.Keyword("filename", f.getCanonicalPath()));
> :
> : try
> : {
> : writer.addDocument(doc); // indexing fails
because
> this statement cannot be executed
> :
> : }
> :
> : catch (Exception e)
> :
> : {
> : System.err.println ("Cannot add doc exception
> thrown!");
> :
> : }
> :
> :
> :
>
//----------------------------------------------------------------------
---------------------
> : // IndexWriter.Java // Lucene 1.4.3
> :
>
//----------------------------------------------------------------------
---------------------
> :
> :
> : public void addDocument(Document doc) throws IOException {
> :
> : addDocument(doc, analyzer);
> : }
> :
> :
> : public void addDocument(Document doc, Analyzer analyzer) throws
> IOException {
> :
> : DocumentWriter dw;
> :
> : dw = new DocumentWriter(ramDirectory, analyzer, similarity,
> maxFieldLength);
> :
> : String segmentName = newSegmentName();
> : dw.addDocument(segmentName, doc); // The program
fails to
> execute this line onwards!
> :
> : synchronized (this) {
> :
> : segmentInfos.addElement(new SegmentInfo(segmentName, 1,
> ramDirectory));
> : maybeMergeSegments();
> : }
> :
> : }
> :
> :
> :
>
//----------------------------------------------------------------------
---------------------
> : // DocumentWriter.Java // Lucene 1.4.3
> :
>
//----------------------------------------------------------------------
---------------------
> :
> :
> :
> : final void addDocument(String segment, Document doc)
> : throws IOException {
> :
> : System.out.println("Entering DocumentWriter.AddDocument....
> (1)");
> :
> : // write field names
> : fieldInfos = new FieldInfos();
> : System.out.println("Entering DocumentWriter.AddDocument....
> (2)");
> :
> : fieldInfos.add(doc);
> : System.out.println("Entering
DocumentWriter.AddDocument....
> (3)");
> :
> : fieldInfos.write(directory, segment + ".fnm");
> :
> : System.out.println("Entering
DocumentWriter.AddDocument....
> (4)"); // The program fails after this
> :
> : // write field values
> : FieldsWriter fieldsWriter =
> : new FieldsWriter(directory, segment, fieldInfos);
//
> Program fails to execute this statement
> :
> : System.out.println("Entering
DocumentWriter.AddDocument....
> (5)");
> :
> : try {
> : fieldsWriter.addDocument(doc);
> :
> : System.out.println("Entering DocumentWriter.AddDocument....
> (6)");
> :
> : } finally {
> : fieldsWriter.close();
> : System.out.println("Entering
DocumentWriter.AddDocument....
> (7)");
> :
> : }
> :
> : System.out.println("Entering
DocumentWriter.AddDocument....
> (8)");
> :
> : // invert doc into postingTable
> : postingTable.clear(); // clear postingTable
> : fieldLengths = new int[fieldInfos.size()]; // init
fieldLengths
> : fieldPositions = new int[fieldInfos.size()]; // init
> fieldPositions
> :
> : System.out.println("Entering
DocumentWriter.AddDocument....
> (9)");
> :
> : fieldBoosts = new float[fieldInfos.size()]; // init
fieldBoosts
> : Arrays.fill(fieldBoosts, doc.getBoost());
> :
> : System.out.println("Entering
DocumentWriter.AddDocument....
> (10)");
> :
> : invertDocument(doc);
> :
> : System.out.println("Entering
DocumentWriter.AddDocument....
> (11)");
> :
> : // sort postingTable into an array
> : Posting[] postings = sortPostingTable();
> :
> : System.out.println("Entering
DocumentWriter.AddDocument....
> (12)");
> :
> : /*
> : for (int i = 0; i < postings.length; i++) {
> : Posting posting = postings[i];
> : System.out.print(posting.term);
> : System.out.print(" freq=" + posting.freq);
> : System.out.print(" pos=");
> : System.out.print(posting.positions[0]);
> : for (int j = 1; j < posting.freq; j++)
> : System.out.print("," + posting.positions[j]);
> : System.out.println("");
> : }
> : */
> :
> : // write postings
> : writePostings(postings, segment);
> :
> : System.out.println("Entering
DocumentWriter.AddDocument....
> (13)");
> :
> : // write norms of indexed fields
> : writeNorms(doc, segment);
> :
> : System.out.println("Entering
DocumentWriter.AddDocument....
> (14)");
> :
> : }
> :
> :
>
//----------------------------------------------------------------------
---------------------
> : // FieldsWriter.Java // Lucene 1.4.3
> :
>
//----------------------------------------------------------------------
---------------------
> :
> :
> : FieldsWriter(Directory d, String segment, FieldInfos fn)
> : throws IOException {
> : fieldInfos = fn;
> : fieldsStream = d.createFile(segment + ".fdt");
> : indexStream = d.createFile(segment + ".fdx");
> : }
> :
> :
>
//----------------------------------------------------------------------
---------------------
> : // FSDirectory.Java // Lucene 1.4.3
> :
>
//----------------------------------------------------------------------
---------------------
> :
> :
> : public final OutputStream createFile(String name) throws IOException
{
> :
> : System.out.println("Entering FSDirectory.createFile....
returning
> an OutputStream");
> :
> : return new FSOutputStream(new File(directory, name));
> : }
> :
> :
---------------------------------------------------------------------
> : To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
> : For additional commands, e-mail: java-user-help@lucene.apache.org
> :
>
>
>
> -Hoss
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
> For additional commands, e-mail: java-user-help@lucene.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
For additional commands, e-mail: java-user-help@lucene.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
For additional commands, e-mail: java-user-help@lucene.apache.org
Re: Lucene 1.4.3 : IndexWriter.addDocument(doc) fails when run on
OS requiring permissions
Posted by Ridzwan Aminuddin <wa...@inbox.com>.
Hi Guys.
Ok thanks for the replies. You guys are right that it is to do with the system and not with Lucene. However, what i'm trying to do is to pinpoint and narrow down to the exact place that causes the system to fail. and then from there try to remedy the problem.
The odd thing is that the program is still able to write other files to the subdirectories that the program itself creates. It is only when it goes through this indexing process that this program halts due to the insufficient permissions. But the directory i provided to store the target index files has been set to read/write/execute (drwxrwxrwx) all permissions.
In any case, i suspect that it is due to this portion of code.
: FieldsWriter(Directory d, String segment, FieldInfos fn)
: throws IOException {
: fieldInfos = fn;
: fieldsStream = d.createFile(segment + ".fdt");
: indexStream = d.createFile(segment + ".fdx");
: }
Are these two files created in the Directory d?
And is this Directory d the ramDirectory that i have provided when i called the method: dw = new DocumentWriter(ramDirectory, analyzer, similarity,
maxFieldLength);
in IndexWriter.Java?
If yes. then where exactly does this ramDirectory point to? cos as far as i can see from the code, ramDirectory is initialised as
RamDirectory ramDirectory = new RamDirectory()
I think the program fails when it tries to create these two files somehow.
Please help to enlighten me?... Maybe knowing the exact path to this ramDirectory would help me in finding out which folder i need to provide access to....
Also, does Lucene ever write any temp data to /tmp ?
> -----Original Message-----
> From: hossman_lucene@fucit.org
> Sent: Thu, 22 Feb 2007 11:43:38 -0800 (PST)
> To: java-user@lucene.apache.org
> Subject: Re: Lucene 1.4.3 : IndexWriter.addDocument(doc) fails when run
> on OS requiring permissions
>
>
> This sounds like it has absolutely nothing to do with Lucene, and
> everything to do with good security permissions -- your Zope/python front
> end is most likely running as a user thta does not have write permissions
> to the directory where your index lives. you'll need to remedy that.
>
> you can write a simple java app that doens't use lucene at all -- just
> creates a file and writes "hellow world" to it -- and you will most
> likely see this exact same behavior, dealing with teh file permissions is
> totally out side the scope of Lucene.
>
>
> : Date: Thu, 22 Feb 2007 00:20:12 -0800
> : From: Ridzwan Aminuddin <wa...@inbox.com>
> : Reply-To: java-user@lucene.apache.org
> : To: java-user@lucene.apache.org
> : Subject: Lucene 1.4.3 : IndexWriter.addDocument(doc) fails when run on
> OS
> : requiring permissions
> :
> : Hi!
> :
> : I'm writing a java program that uses Lucene 1.4.3 to index and create a
> vector file of words found in Text Files. The purpose is for text mining.
> :
> : I created a Java .Jar file from my program and my python script calls
> the Java Jar executable. This is all triggered by my DTML code.
> :
> : I'm running on Linux and i have no problem executing the script when i
> execute via command line. But once i trigger the script via the web
> (using Zope/Plone external methods ) it doesn't work anymore. This is
> because of the strict permissions that LInux has over its files and
> folders.
> :
> : I've narrowed down the problem to the IndexWriter.addDocument(doc)
> method in Lucene 1.4.3 and as you can see below my code fails
> specifically when a new FieldsWriter object is being initialised.
> :
> : I strongly suspect that it fails at this point but have no idea how to
> overcome this problem. I know that it has to do with the permissions
> because th eprogram works like a miracle when it is called via command
> line by the super user (sudo).
> :
> : Could anyone give me any pointers or ideas of how i could overcome
> this.
> :
> : The final statement which is printed before the program hangs is:
> : "Entering DocumentWriter.AddDocument.... (4)"
> :
> : Here is the portions of my relevant code :
> :
> :
> :
> :
> //-------------------------------------------------------------------------------------------
> : // Indexer.Java // This is my own method and class
> :
> //-------------------------------------------------------------------------------------------
> : // continued from some other code......
> :
> : Document doc = new Document();
> :
> : doc.add(Field.Text("articleTitle", articleTitle, true));
> : doc.add(Field.Text("articleURL", articleURL, true));
> : doc.add(Field.Text("articleSummary", articleSummary, true));
> : doc.add(Field.Text("articleDate", articleDate, true));
> : doc.add(Field.Text("articleSource", articleSource, true));
> : doc.add(Field.Text("articleBody", articleBody, true));
> : doc.add(Field.Keyword("filename", f.getCanonicalPath()));
> :
> : try
> : {
> : writer.addDocument(doc); // indexing fails because
> this statement cannot be executed
> :
> : }
> :
> : catch (Exception e)
> :
> : {
> : System.err.println ("Cannot add doc exception
> thrown!");
> :
> : }
> :
> :
> :
> //-------------------------------------------------------------------------------------------
> : // IndexWriter.Java // Lucene 1.4.3
> :
> //-------------------------------------------------------------------------------------------
> :
> :
> : public void addDocument(Document doc) throws IOException {
> :
> : addDocument(doc, analyzer);
> : }
> :
> :
> : public void addDocument(Document doc, Analyzer analyzer) throws
> IOException {
> :
> : DocumentWriter dw;
> :
> : dw = new DocumentWriter(ramDirectory, analyzer, similarity,
> maxFieldLength);
> :
> : String segmentName = newSegmentName();
> : dw.addDocument(segmentName, doc); // The program fails to
> execute this line onwards!
> :
> : synchronized (this) {
> :
> : segmentInfos.addElement(new SegmentInfo(segmentName, 1,
> ramDirectory));
> : maybeMergeSegments();
> : }
> :
> : }
> :
> :
> :
> //-------------------------------------------------------------------------------------------
> : // DocumentWriter.Java // Lucene 1.4.3
> :
> //-------------------------------------------------------------------------------------------
> :
> :
> :
> : final void addDocument(String segment, Document doc)
> : throws IOException {
> :
> : System.out.println("Entering DocumentWriter.AddDocument....
> (1)");
> :
> : // write field names
> : fieldInfos = new FieldInfos();
> : System.out.println("Entering DocumentWriter.AddDocument....
> (2)");
> :
> : fieldInfos.add(doc);
> : System.out.println("Entering DocumentWriter.AddDocument....
> (3)");
> :
> : fieldInfos.write(directory, segment + ".fnm");
> :
> : System.out.println("Entering DocumentWriter.AddDocument....
> (4)"); // The program fails after this
> :
> : // write field values
> : FieldsWriter fieldsWriter =
> : new FieldsWriter(directory, segment, fieldInfos); //
> Program fails to execute this statement
> :
> : System.out.println("Entering DocumentWriter.AddDocument....
> (5)");
> :
> : try {
> : fieldsWriter.addDocument(doc);
> :
> : System.out.println("Entering DocumentWriter.AddDocument....
> (6)");
> :
> : } finally {
> : fieldsWriter.close();
> : System.out.println("Entering DocumentWriter.AddDocument....
> (7)");
> :
> : }
> :
> : System.out.println("Entering DocumentWriter.AddDocument....
> (8)");
> :
> : // invert doc into postingTable
> : postingTable.clear(); // clear postingTable
> : fieldLengths = new int[fieldInfos.size()]; // init fieldLengths
> : fieldPositions = new int[fieldInfos.size()]; // init
> fieldPositions
> :
> : System.out.println("Entering DocumentWriter.AddDocument....
> (9)");
> :
> : fieldBoosts = new float[fieldInfos.size()]; // init fieldBoosts
> : Arrays.fill(fieldBoosts, doc.getBoost());
> :
> : System.out.println("Entering DocumentWriter.AddDocument....
> (10)");
> :
> : invertDocument(doc);
> :
> : System.out.println("Entering DocumentWriter.AddDocument....
> (11)");
> :
> : // sort postingTable into an array
> : Posting[] postings = sortPostingTable();
> :
> : System.out.println("Entering DocumentWriter.AddDocument....
> (12)");
> :
> : /*
> : for (int i = 0; i < postings.length; i++) {
> : Posting posting = postings[i];
> : System.out.print(posting.term);
> : System.out.print(" freq=" + posting.freq);
> : System.out.print(" pos=");
> : System.out.print(posting.positions[0]);
> : for (int j = 1; j < posting.freq; j++)
> : System.out.print("," + posting.positions[j]);
> : System.out.println("");
> : }
> : */
> :
> : // write postings
> : writePostings(postings, segment);
> :
> : System.out.println("Entering DocumentWriter.AddDocument....
> (13)");
> :
> : // write norms of indexed fields
> : writeNorms(doc, segment);
> :
> : System.out.println("Entering DocumentWriter.AddDocument....
> (14)");
> :
> : }
> :
> :
> //-------------------------------------------------------------------------------------------
> : // FieldsWriter.Java // Lucene 1.4.3
> :
> //-------------------------------------------------------------------------------------------
> :
> :
> : FieldsWriter(Directory d, String segment, FieldInfos fn)
> : throws IOException {
> : fieldInfos = fn;
> : fieldsStream = d.createFile(segment + ".fdt");
> : indexStream = d.createFile(segment + ".fdx");
> : }
> :
> :
> //-------------------------------------------------------------------------------------------
> : // FSDirectory.Java // Lucene 1.4.3
> :
> //-------------------------------------------------------------------------------------------
> :
> :
> : public final OutputStream createFile(String name) throws IOException {
> :
> : System.out.println("Entering FSDirectory.createFile.... returning
> an OutputStream");
> :
> : return new FSOutputStream(new File(directory, name));
> : }
> :
> : ---------------------------------------------------------------------
> : To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
> : For additional commands, e-mail: java-user-help@lucene.apache.org
> :
>
>
>
> -Hoss
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
> For additional commands, e-mail: java-user-help@lucene.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
For additional commands, e-mail: java-user-help@lucene.apache.org
Re: Lucene 1.4.3 : IndexWriter.addDocument(doc) fails when run on
OS requiring permissions
Posted by Chris Hostetter <ho...@fucit.org>.
This sounds like it has absolutely nothing to do with Lucene, and
everything to do with good security permissions -- your Zope/python front
end is most likely running as a user thta does not have write permissions
to the directory where your index lives. you'll need to remedy that.
you can write a simple java app that doens't use lucene at all -- just
creates a file and writes "hellow world" to it -- and you will most
likely see this exact same behavior, dealing with teh file permissions is
totally out side the scope of Lucene.
: Date: Thu, 22 Feb 2007 00:20:12 -0800
: From: Ridzwan Aminuddin <wa...@inbox.com>
: Reply-To: java-user@lucene.apache.org
: To: java-user@lucene.apache.org
: Subject: Lucene 1.4.3 : IndexWriter.addDocument(doc) fails when run on OS
: requiring permissions
:
: Hi!
:
: I'm writing a java program that uses Lucene 1.4.3 to index and create a vector file of words found in Text Files. The purpose is for text mining.
:
: I created a Java .Jar file from my program and my python script calls the Java Jar executable. This is all triggered by my DTML code.
:
: I'm running on Linux and i have no problem executing the script when i execute via command line. But once i trigger the script via the web (using Zope/Plone external methods ) it doesn't work anymore. This is because of the strict permissions that LInux has over its files and folders.
:
: I've narrowed down the problem to the IndexWriter.addDocument(doc) method in Lucene 1.4.3 and as you can see below my code fails specifically when a new FieldsWriter object is being initialised.
:
: I strongly suspect that it fails at this point but have no idea how to overcome this problem. I know that it has to do with the permissions because th eprogram works like a miracle when it is called via command line by the super user (sudo).
:
: Could anyone give me any pointers or ideas of how i could overcome this.
:
: The final statement which is printed before the program hangs is:
: "Entering DocumentWriter.AddDocument.... (4)"
:
: Here is the portions of my relevant code :
:
:
:
: //-------------------------------------------------------------------------------------------
: // Indexer.Java // This is my own method and class
: //-------------------------------------------------------------------------------------------
: // continued from some other code......
:
: Document doc = new Document();
:
: doc.add(Field.Text("articleTitle", articleTitle, true));
: doc.add(Field.Text("articleURL", articleURL, true));
: doc.add(Field.Text("articleSummary", articleSummary, true));
: doc.add(Field.Text("articleDate", articleDate, true));
: doc.add(Field.Text("articleSource", articleSource, true));
: doc.add(Field.Text("articleBody", articleBody, true));
: doc.add(Field.Keyword("filename", f.getCanonicalPath()));
:
: try
: {
: writer.addDocument(doc); // indexing fails because this statement cannot be executed
:
: }
:
: catch (Exception e)
:
: {
: System.err.println ("Cannot add doc exception thrown!");
:
: }
:
:
: //-------------------------------------------------------------------------------------------
: // IndexWriter.Java // Lucene 1.4.3
: //-------------------------------------------------------------------------------------------
:
:
: public void addDocument(Document doc) throws IOException {
:
: addDocument(doc, analyzer);
: }
:
:
: public void addDocument(Document doc, Analyzer analyzer) throws IOException {
:
: DocumentWriter dw;
:
: dw = new DocumentWriter(ramDirectory, analyzer, similarity, maxFieldLength);
:
: String segmentName = newSegmentName();
: dw.addDocument(segmentName, doc); // The program fails to execute this line onwards!
:
: synchronized (this) {
:
: segmentInfos.addElement(new SegmentInfo(segmentName, 1, ramDirectory));
: maybeMergeSegments();
: }
:
: }
:
:
: //-------------------------------------------------------------------------------------------
: // DocumentWriter.Java // Lucene 1.4.3
: //-------------------------------------------------------------------------------------------
:
:
:
: final void addDocument(String segment, Document doc)
: throws IOException {
:
: System.out.println("Entering DocumentWriter.AddDocument.... (1)");
:
: // write field names
: fieldInfos = new FieldInfos();
: System.out.println("Entering DocumentWriter.AddDocument.... (2)");
:
: fieldInfos.add(doc);
: System.out.println("Entering DocumentWriter.AddDocument.... (3)");
:
: fieldInfos.write(directory, segment + ".fnm");
:
: System.out.println("Entering DocumentWriter.AddDocument.... (4)"); // The program fails after this
:
: // write field values
: FieldsWriter fieldsWriter =
: new FieldsWriter(directory, segment, fieldInfos); // Program fails to execute this statement
:
: System.out.println("Entering DocumentWriter.AddDocument.... (5)");
:
: try {
: fieldsWriter.addDocument(doc);
:
: System.out.println("Entering DocumentWriter.AddDocument.... (6)");
:
: } finally {
: fieldsWriter.close();
: System.out.println("Entering DocumentWriter.AddDocument.... (7)");
:
: }
:
: System.out.println("Entering DocumentWriter.AddDocument.... (8)");
:
: // invert doc into postingTable
: postingTable.clear(); // clear postingTable
: fieldLengths = new int[fieldInfos.size()]; // init fieldLengths
: fieldPositions = new int[fieldInfos.size()]; // init fieldPositions
:
: System.out.println("Entering DocumentWriter.AddDocument.... (9)");
:
: fieldBoosts = new float[fieldInfos.size()]; // init fieldBoosts
: Arrays.fill(fieldBoosts, doc.getBoost());
:
: System.out.println("Entering DocumentWriter.AddDocument.... (10)");
:
: invertDocument(doc);
:
: System.out.println("Entering DocumentWriter.AddDocument.... (11)");
:
: // sort postingTable into an array
: Posting[] postings = sortPostingTable();
:
: System.out.println("Entering DocumentWriter.AddDocument.... (12)");
:
: /*
: for (int i = 0; i < postings.length; i++) {
: Posting posting = postings[i];
: System.out.print(posting.term);
: System.out.print(" freq=" + posting.freq);
: System.out.print(" pos=");
: System.out.print(posting.positions[0]);
: for (int j = 1; j < posting.freq; j++)
: System.out.print("," + posting.positions[j]);
: System.out.println("");
: }
: */
:
: // write postings
: writePostings(postings, segment);
:
: System.out.println("Entering DocumentWriter.AddDocument.... (13)");
:
: // write norms of indexed fields
: writeNorms(doc, segment);
:
: System.out.println("Entering DocumentWriter.AddDocument.... (14)");
:
: }
:
: //-------------------------------------------------------------------------------------------
: // FieldsWriter.Java // Lucene 1.4.3
: //-------------------------------------------------------------------------------------------
:
:
: FieldsWriter(Directory d, String segment, FieldInfos fn)
: throws IOException {
: fieldInfos = fn;
: fieldsStream = d.createFile(segment + ".fdt");
: indexStream = d.createFile(segment + ".fdx");
: }
:
: //-------------------------------------------------------------------------------------------
: // FSDirectory.Java // Lucene 1.4.3
: //-------------------------------------------------------------------------------------------
:
:
: public final OutputStream createFile(String name) throws IOException {
:
: System.out.println("Entering FSDirectory.createFile.... returning an OutputStream");
:
: return new FSOutputStream(new File(directory, name));
: }
:
: ---------------------------------------------------------------------
: To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
: For additional commands, e-mail: java-user-help@lucene.apache.org
:
-Hoss
---------------------------------------------------------------------
To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
For additional commands, e-mail: java-user-help@lucene.apache.org
Re: Lucene 1.4.3 : IndexWriter.addDocument(doc) fails when run on
OS requiring permissions
Posted by "Michael D. Curtin" <mi...@curtin.com>.
Is your disk almost full? Under Linux, when you reach about 90% used on
a file system, only the superuser can allocate more space (e.g. create
files, add data to files, etc.).
--MDC
---------------------------------------------------------------------
To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
For additional commands, e-mail: java-user-help@lucene.apache.org
Re: Lucene 1.4.3 : IndexWriter.addDocument(doc) fails when run on OS
requiring permissions
Posted by Doron Cohen <DO...@il.ibm.com>.
This is a very common use case and Lucene is most likely not
the problem cause.
My guess is that (1) the first attempt to write anything to
disk failed. (2) opening the IndexWriter succeeded because
(a) the index exists already (from previous successful run) and
(b) locks are maintained in /tmp or so.
You can try running the same way some simple code that just
writes something to a file in the same directory where the
index is maintained (also, check access permissions at that folder).
Hope this helps,
Doron
(BTW, an exception stack trace would be far more informative than
the print statements here - best catch the exception in the
top level and print it.)
Ridzwan Aminuddin <wa...@inbox.com> wrote on 22/02/2007 00:20:12:
> Hi!
>
> I'm writing a java program that uses Lucene 1.4.3 to index and
> create a vector file of words found in Text Files. The purpose is
> for text mining.
>
> I created a Java .Jar file from my program and my python script
> calls the Java Jar executable. This is all triggered by my DTML code.
>
> I'm running on Linux and i have no problem executing the script when
> i execute via command line. But once i trigger the script via the
> web (using Zope/Plone external methods ) it doesn't work anymore.
> This is because of the strict permissions that LInux has over its
> files and folders.
>
> I've narrowed down the problem to the IndexWriter.addDocument(doc)
> method in Lucene 1.4.3 and as you can see below my code fails
> specifically when a new FieldsWriter object is being initialised.
>
> I strongly suspect that it fails at this point but have no idea how
> to overcome this problem. I know that it has to do with the
> permissions because th eprogram works like a miracle when it is
> called via command line by the super user (sudo).
>
> Could anyone give me any pointers or ideas of how i could overcome this.
>
> The final statement which is printed before the program hangs is:
> "Entering DocumentWriter.AddDocument.... (4)"
>
> Here is the portions of my relevant code :
>
>
>
>
//-------------------------------------------------------------------------------------------
> // Indexer.Java // This is my own method and class
>
//-------------------------------------------------------------------------------------------
> // continued from some other code......
>
> Document doc = new Document();
>
> doc.add(Field.Text("articleTitle", articleTitle, true));
> doc.add(Field.Text("articleURL", articleURL, true));
> doc.add(Field.Text("articleSummary", articleSummary, true));
> doc.add(Field.Text("articleDate", articleDate, true));
> doc.add(Field.Text("articleSource", articleSource, true));
> doc.add(Field.Text("articleBody", articleBody, true));
> doc.add(Field.Keyword("filename", f.getCanonicalPath()));
>
> try
> {
> writer.addDocument(doc); // indexing fails
> because this statement cannot be executed
>
> }
>
> catch (Exception e)
>
> {
> System.err.println ("Cannot add doc exception
thrown!");
>
> }
>
>
>
//-------------------------------------------------------------------------------------------
> // IndexWriter.Java // Lucene 1.4.3
>
//-------------------------------------------------------------------------------------------
>
>
> public void addDocument(Document doc) throws IOException {
>
> addDocument(doc, analyzer);
> }
>
>
> public void addDocument(Document doc, Analyzer analyzer) throws
IOException {
>
> DocumentWriter dw;
>
> dw = new DocumentWriter(ramDirectory, analyzer, similarity,
> maxFieldLength);
>
> String segmentName = newSegmentName();
> dw.addDocument(segmentName, doc); // The program
> fails to execute this line onwards!
>
> synchronized (this) {
>
> segmentInfos.addElement(new SegmentInfo(segmentName, 1,
ramDirectory));
> maybeMergeSegments();
> }
>
> }
>
>
>
//-------------------------------------------------------------------------------------------
> // DocumentWriter.Java // Lucene 1.4.3
>
//-------------------------------------------------------------------------------------------
>
>
>
> final void addDocument(String segment, Document doc)
> throws IOException {
>
> System.out.println("Entering DocumentWriter.AddDocument.... (1)");
>
> // write field names
> fieldInfos = new FieldInfos();
> System.out.println("Entering DocumentWriter.AddDocument.... (2)");
>
> fieldInfos.add(doc);
> System.out.println("Entering DocumentWriter.AddDocument....
(3)");
>
> fieldInfos.write(directory, segment + ".fnm");
>
> System.out.println("Entering DocumentWriter.
> AddDocument.... (4)"); // The program fails after this
>
> // write field values
> FieldsWriter fieldsWriter =
> new FieldsWriter(directory, segment, fieldInfos);
> // Program fails to execute this statement
>
> System.out.println("Entering DocumentWriter.AddDocument....
(5)");
>
> try {
> fieldsWriter.addDocument(doc);
>
> System.out.println("Entering DocumentWriter.AddDocument.... (6)");
>
> } finally {
> fieldsWriter.close();
> System.out.println("Entering DocumentWriter.AddDocument....
(7)");
>
> }
>
> System.out.println("Entering DocumentWriter.AddDocument....
(8)");
>
> // invert doc into postingTable
> postingTable.clear(); // clear postingTable
> fieldLengths = new int[fieldInfos.size()]; // init fieldLengths
> fieldPositions = new int[fieldInfos.size()]; // init fieldPositions
>
> System.out.println("Entering DocumentWriter.AddDocument....
(9)");
>
> fieldBoosts = new float[fieldInfos.size()]; // init fieldBoosts
> Arrays.fill(fieldBoosts, doc.getBoost());
>
> System.out.println("Entering DocumentWriter.AddDocument....
(10)");
>
> invertDocument(doc);
>
> System.out.println("Entering DocumentWriter.AddDocument....
(11)");
>
> // sort postingTable into an array
> Posting[] postings = sortPostingTable();
>
> System.out.println("Entering DocumentWriter.AddDocument....
(12)");
>
> /*
> for (int i = 0; i < postings.length; i++) {
> Posting posting = postings[i];
> System.out.print(posting.term);
> System.out.print(" freq=" + posting.freq);
> System.out.print(" pos=");
> System.out.print(posting.positions[0]);
> for (int j = 1; j < posting.freq; j++)
> System.out.print("," + posting.positions[j]);
> System.out.println("");
> }
> */
>
> // write postings
> writePostings(postings, segment);
>
> System.out.println("Entering DocumentWriter.AddDocument....
(13)");
>
> // write norms of indexed fields
> writeNorms(doc, segment);
>
> System.out.println("Entering DocumentWriter.AddDocument....
(14)");
>
> }
>
>
//-------------------------------------------------------------------------------------------
> // FieldsWriter.Java // Lucene 1.4.3
>
//-------------------------------------------------------------------------------------------
>
>
> FieldsWriter(Directory d, String segment, FieldInfos fn)
> throws IOException {
> fieldInfos = fn;
> fieldsStream = d.createFile(segment + ".fdt");
> indexStream = d.createFile(segment + ".fdx");
> }
>
>
//-------------------------------------------------------------------------------------------
> // FSDirectory.Java // Lucene 1.4.3
>
//-------------------------------------------------------------------------------------------
>
>
> public final OutputStream createFile(String name) throws IOException {
>
> System.out.println("Entering FSDirectory.createFile....
> returning an OutputStream");
>
> return new FSOutputStream(new File(directory, name));
> }
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
> For additional commands, e-mail: java-user-help@lucene.apache.org
>
---------------------------------------------------------------------
To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org
For additional commands, e-mail: java-user-help@lucene.apache.org