You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by jo...@apache.org on 2011/09/26 12:58:12 UTC
svn commit: r1175787 - in
/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server:
./ search/ store/
Author: joern
Date: Mon Sep 26 10:58:11 2011
New Revision: 1175787
URL: http://svn.apache.org/viewvc?rev=1175787&view=rev
Log:
OPENNLP-299 Index mapping can now be defined per corpus.
Modified:
incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorporaResource.java
incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/LuceneSearchService.java
incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/CorporaStore.java
incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/CorpusStore.java
incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/DerbyCorporaStore.java
incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/DerbyCorpusStore.java
incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/MemoryCorporaStore.java
incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/MemoryCorpusStore.java
Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorporaResource.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorporaResource.java?rev=1175787&r1=1175786&r2=1175787&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorporaResource.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/CorporaResource.java Mon Sep 26 10:58:11 2011
@@ -27,6 +27,7 @@ import javax.ws.rs.QueryParam;
import javax.ws.rs.core.MediaType;
import org.apache.opennlp.corpus_server.store.CorporaStore;
+import org.codehaus.jettison.json.JSONArray;
@Path("/corpora")
public class CorporaResource {
@@ -40,15 +41,19 @@ public class CorporaResource {
* @param typeSystemBytes
*/
@POST
- @Consumes(MediaType.TEXT_XML)
+ @Consumes(MediaType.APPLICATION_JSON)
@Path("_createCorpus")
public void createCorpus(@QueryParam("corpusName") String corpusName,
- byte[] typeSystemBytes) throws IOException {
+ byte[][] resources) throws IOException {
+
+ if (resources.length != 2) {
+ // TODO: throw exception
+ }
CorpusServer corpusServer = CorpusServer.getInstance();
CorporaStore store = corpusServer.getStore();
- store.createCorpus(corpusName, typeSystemBytes);
+ store.createCorpus(corpusName, resources[0], resources[1]);
}
@Path("{corpus}")
Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/LuceneSearchService.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/LuceneSearchService.java?rev=1175787&r1=1175786&r2=1175787&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/LuceneSearchService.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/search/LuceneSearchService.java Mon Sep 26 10:58:11 2011
@@ -42,6 +42,7 @@ import org.apache.lucene.search.Collecto
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
+import org.apache.opennlp.corpus_server.CorpusServer;
import org.apache.opennlp.corpus_server.UimaUtil;
import org.apache.opennlp.corpus_server.store.CorporaStore;
import org.apache.opennlp.corpus_server.store.CorpusStore;
@@ -78,10 +79,11 @@ public class LuceneSearchService impleme
}
private void createIndexWriter(String corpusId, boolean createIndex) throws IOException {
+
// Set the index mapping file for this corpus in the analysis engine descriptor
+ CorpusStore corpusStore = CorpusServer.getInstance().getStore().getCorpus(corpusId);
- XMLInputSource in = new XMLInputSource(LuceneSearchService.class.getResourceAsStream(
- "/org/apache/opennlp/corpus_server/search/LuceneIndexer.xml"), new File(""));
+ XMLInputSource in = new XMLInputSource(new ByteArrayInputStream(corpusStore.getIndexMapping()), new File(""));
try {
AnalysisEngineDescription specifier;
Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/CorporaStore.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/CorporaStore.java?rev=1175787&r1=1175786&r2=1175787&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/CorporaStore.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/CorporaStore.java Mon Sep 26 10:58:11 2011
@@ -43,8 +43,10 @@ public interface CorporaStore {
*
* @param corpusName
* @param typeSystemBytes
+ * @param indexMapping
*/
- void createCorpus(String corpusName, byte typeSystemBytes[]) throws IOException;
+ void createCorpus(String corpusName, byte typeSystemBytes[],
+ byte indexMapping[]) throws IOException;
Set<String> getCorpusIds() throws IOException;
Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/CorpusStore.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/CorpusStore.java?rev=1175787&r1=1175786&r2=1175787&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/CorpusStore.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/CorpusStore.java Mon Sep 26 10:58:11 2011
@@ -36,4 +36,6 @@ public interface CorpusStore {
void updateCAS(String casID, byte[] content) throws IOException;
TypeSystemDescription getTypeSystem() throws IOException;
+
+ byte[] getIndexMapping() throws IOException;
}
Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/DerbyCorporaStore.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/DerbyCorporaStore.java?rev=1175787&r1=1175786&r2=1175787&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/DerbyCorporaStore.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/DerbyCorporaStore.java Mon Sep 26 10:58:11 2011
@@ -54,7 +54,7 @@ public class DerbyCorporaStore extends A
}
@Override
- public void createCorpus(String corpusName, byte[] typeSystemBytes)
+ public void createCorpus(String corpusName, byte[] typeSystemBytes, byte indexMapping[])
throws IOException {
try {
@@ -62,22 +62,36 @@ public class DerbyCorporaStore extends A
Statement s = conn.createStatement();
s.execute("create table " + corpusName +
"(name varchar(1024), cas blob, unique (name))");
-
+
+ s.close();
+
// Insert the type system
- PreparedStatement ps = conn.prepareStatement("insert into " + corpusName
+ PreparedStatement typeSystemPS = conn.prepareStatement("insert into " + corpusName
+ " values (?, ?)");
- ps.setString(1, "_typesystem");
+ typeSystemPS.setString(1, "_typesystem");
- Blob b = conn.createBlob();
- b.setBytes(1, typeSystemBytes);
- ps.setBlob(2, b);
+ Blob typeSystemBlob = conn.createBlob();
+ typeSystemBlob.setBytes(1, typeSystemBytes);
+ typeSystemPS.setBlob(2, typeSystemBlob);
- ps.executeUpdate();
+ typeSystemPS.executeUpdate();
+ PreparedStatement indexMappingPS = conn.prepareStatement("insert into " + corpusName
+ + " values (?, ?)");
+
+ indexMappingPS.setString(1, "_indexMapping");
+
+ Blob indexMappingBlob = conn.createBlob();
+ indexMappingBlob.setBytes(1, indexMapping);
+ indexMappingPS.setBlob(2, indexMappingBlob);
+
+ indexMappingPS.executeUpdate();
+
conn.commit();
- ps.close();
+ typeSystemPS.close();
+ indexMappingPS.close();
conn.close();
} catch (SQLException e) {
Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/DerbyCorpusStore.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/DerbyCorpusStore.java?rev=1175787&r1=1175786&r2=1175787&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/DerbyCorpusStore.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/DerbyCorpusStore.java Mon Sep 26 10:58:11 2011
@@ -188,4 +188,33 @@ public class DerbyCorpusStore implements
return tsDescription;
}
+
+ @Override
+ public byte[] getIndexMapping() throws IOException {
+ byte indexMappingBytes[] = null;
+
+ try {
+ Connection conn = dataSource.getConnection();
+ Statement s = conn.createStatement();
+ ResultSet indexMappingResult = s.executeQuery("select * FROM " + corpusName +
+ " WHERE name='_indexMapping'");
+
+ if (indexMappingResult.next()) {
+ indexMappingBytes = indexMappingResult.getBytes(2);
+ }
+
+ indexMappingResult.close();
+ s.close();
+ conn.close();
+ } catch (SQLException e) {
+
+ if (LOGGER.isLoggable(Level.SEVERE)) {
+ LOGGER.log(Level.SEVERE, "Failed to retrieve type system", e);
+ }
+
+ throw new IOException(e);
+ }
+
+ return indexMappingBytes;
+ }
}
Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/MemoryCorporaStore.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/MemoryCorporaStore.java?rev=1175787&r1=1175786&r2=1175787&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/MemoryCorporaStore.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/MemoryCorporaStore.java Mon Sep 26 10:58:11 2011
@@ -54,10 +54,10 @@ public class MemoryCorporaStore extends
// Note: Add one twice, overwrites an existing one!
public void createCorpus(String corpusName,
- byte typeSystemBytes[]) {
+ byte typeSystemBytes[], byte indexMapping[]) {
corpora.put(corpusName, new MemoryCorpusStore(corpusName,
UimaUtil.createTypeSystemDescription(
- new ByteArrayInputStream(typeSystemBytes))));
+ new ByteArrayInputStream(typeSystemBytes)), indexMapping));
}
public MemoryCorpusStore getCorpus(String corpusId) {
Modified: incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/MemoryCorpusStore.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/MemoryCorpusStore.java?rev=1175787&r1=1175786&r2=1175787&view=diff
==============================================================================
--- incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/MemoryCorpusStore.java (original)
+++ incubator/opennlp/sandbox/corpus-server/src/main/java/org/apache/opennlp/corpus_server/store/MemoryCorpusStore.java Mon Sep 26 10:58:11 2011
@@ -33,12 +33,14 @@ public class MemoryCorpusStore implement
private final String corpusName;
private final TypeSystemDescription typeSystem;
+ private byte[] indexMapping;
private Map<String, byte[]> casStore = new HashMap<String, byte[]>();
- MemoryCorpusStore(String corpusName, TypeSystemDescription typeSystem) {
+ MemoryCorpusStore(String corpusName, TypeSystemDescription typeSystem, byte[] indexMapping) {
this.corpusName = corpusName;
this.typeSystem = typeSystem;
+ this.indexMapping = indexMapping;
}
@Override
@@ -76,4 +78,9 @@ public class MemoryCorpusStore implement
public TypeSystemDescription getTypeSystem() {
return typeSystem;
}
+
+ @Override
+ public byte[] getIndexMapping() throws IOException {
+ return indexMapping;
+ }
}