You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by bg...@apache.org on 2012/04/06 00:53:14 UTC
svn commit: r1310122 - in /opennlp/sandbox/opennlp-similarity: pom.xml
src/main/java/opennlp/tools/textsimilarity/chunker2matcher/ParserCacheSerializer.java
src/main/java/opennlp/tools/textsimilarity/chunker2matcher/ParserChunker2MatcherProcessor.java
Author: bgalitsky
Date: Thu Apr 5 22:53:14 2012
New Revision: 1310122
URL: http://svn.apache.org/viewvc?rev=1310122&view=rev
Log:
OPENNLP-420: cached parsing results for junits *.dat file
now caches into CSV file instead of java object serialization
Modified:
opennlp/sandbox/opennlp-similarity/pom.xml
opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/ParserCacheSerializer.java
opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/ParserChunker2MatcherProcessor.java
Modified: opennlp/sandbox/opennlp-similarity/pom.xml
URL: http://svn.apache.org/viewvc/opennlp/sandbox/opennlp-similarity/pom.xml?rev=1310122&r1=1310121&r2=1310122&view=diff
==============================================================================
--- opennlp/sandbox/opennlp-similarity/pom.xml (original)
+++ opennlp/sandbox/opennlp-similarity/pom.xml Thu Apr 5 22:53:14 2012
@@ -75,6 +75,11 @@
<artifactId>xstream</artifactId>
<version>1.4.2</version>
</dependency>
+ <dependency>
+ <groupId>net.sf.opencsv</groupId>
+ <artifactId>opencsv</artifactId>
+ <version>2.0</version>
+ </dependency>
</dependencies>
<build>
Modified: opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/ParserCacheSerializer.java
URL: http://svn.apache.org/viewvc/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/ParserCacheSerializer.java?rev=1310122&r1=1310121&r2=1310122&view=diff
==============================================================================
--- opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/ParserCacheSerializer.java (original)
+++ opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/ParserCacheSerializer.java Thu Apr 5 22:53:14 2012
@@ -34,64 +34,113 @@
package opennlp.tools.textsimilarity.chunker2matcher;
import java.io.FileInputStream;
+import java.io.FileNotFoundException;
import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
import java.util.Map;
+import java.util.logging.Logger;
+import au.com.bytecode.opencsv.CSVReader;
+import au.com.bytecode.opencsv.CSVWriter;
public class ParserCacheSerializer {
- private static String RESOURCE_DIR = "resources/";
- public static String parseCacheFileName = "sentence_parseObject.dat";
-
- public static void writeObject(Object objectToSerialize)
- {
- String filename = RESOURCE_DIR + parseCacheFileName;
- FileOutputStream fos = null;
- ObjectOutputStream out = null;
- try
- {
- fos = new FileOutputStream(filename);
- out = new ObjectOutputStream(fos);
- out.writeObject(objectToSerialize);
- out.close();
- }
- catch (IOException ex)
- {
- ex.printStackTrace();
- }
-
- }
-
- public static Object readObject()
- {
- String filename = RESOURCE_DIR + parseCacheFileName;
- Object data = null;
- FileInputStream fis = null;
- ObjectInputStream in = null;
- try
- {
- fis = new FileInputStream(filename);
- in = new ObjectInputStream(fis);
- data = (Object) in.readObject();
- in.close();
- }
- catch (IOException ex)
- {
- System.out.println("Cant find parsing cache file ");
- }
- catch (ClassNotFoundException ex)
- {
- ex.printStackTrace();
- }
-
- return data;
-
- }
-
- public class ParserObjectSer{
-
- }
+ private static Logger LOG = Logger.getLogger("opennlp.tools.textsimilarity.chunker2matcher.ParserCacheSerializer");
+ private static boolean javaObjectSerialization = false;
+ private static String RESOURCE_DIR = "src/test/resources/";
+ public static String parseCacheFileName = "sentence_parseObject.dat";
+ public static String parseCacheFileNameCSV = "sentence_parseObject.csv";
+
+ public static void writeObject(Object objectToSerialize) {
+ if (javaObjectSerialization) {
+ String filename = RESOURCE_DIR + parseCacheFileName;
+ FileOutputStream fos = null;
+ ObjectOutputStream out = null;
+ try {
+ fos = new FileOutputStream(filename);
+ out = new ObjectOutputStream(fos);
+ out.writeObject(objectToSerialize);
+ out.close();
+ } catch (IOException ex) {
+ ex.printStackTrace();
+ }
+ } else {
+
+ Map<String, String[][]> sentence_parseObject = (Map<String, String[][]>) objectToSerialize;
+ List<String> keys = new ArrayList<String>(sentence_parseObject.keySet());
+ try {
+ CSVWriter writer = new CSVWriter(new FileWriter(RESOURCE_DIR
+ + parseCacheFileNameCSV, false));
+ for (String k : keys) {
+ String[][] triplet = sentence_parseObject.get(k);
+ writer.writeNext(new String[] { k });
+ writer.writeNext(triplet[0]);
+ writer.writeNext(triplet[1]);
+ writer.writeNext(triplet[2]);
+
+ }
+ writer.close();
+ } catch (IOException e) {
+ LOG.severe(e.getMessage());
+ }
+ }
+
+ }
+
+ public static Object readObject() {
+ if (javaObjectSerialization) {
+ String filename = RESOURCE_DIR + parseCacheFileName;
+ Object data = null;
+ FileInputStream fis = null;
+ ObjectInputStream in = null;
+ try {
+ fis = new FileInputStream(filename);
+ in = new ObjectInputStream(fis);
+ data = (Object) in.readObject();
+ in.close();
+ } catch (IOException ex) {
+ System.out.println("Cant find parsing cache file ");
+ } catch (ClassNotFoundException ex) {
+ ex.printStackTrace();
+ }
+ return data;
+ } else {
+ CSVReader reader = null;
+ List<String[]> lines = null;
+
+ try {
+ reader = new CSVReader(
+ new FileReader(RESOURCE_DIR + parseCacheFileNameCSV), ',');
+ lines = reader.readAll();
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ return null;
+ } catch (IOException ioe) {
+ ioe.printStackTrace();
+ return null;
+ }
+ Map<String, String[][]> sentence_parseObject = new HashMap<String, String[][]>();
+ int count = 0;
+ for (int i = 0; i < lines.size() - 3; i += 4) {
+ String key = lines.get(i)[0];
+ String[][] value = new String[][] { lines.get(i + 1), lines.get(i + 2),
+ lines.get(i + 3) };
+ sentence_parseObject.put(key, value);
+ }
+
+ return sentence_parseObject;
+ }
+
+ }
+
+ public class ParserObjectSer {
+
+ }
}
Modified: opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/ParserChunker2MatcherProcessor.java
URL: http://svn.apache.org/viewvc/opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/ParserChunker2MatcherProcessor.java?rev=1310122&r1=1310121&r2=1310122&view=diff
==============================================================================
--- opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/ParserChunker2MatcherProcessor.java (original)
+++ opennlp/sandbox/opennlp-similarity/src/main/java/opennlp/tools/textsimilarity/chunker2matcher/ParserChunker2MatcherProcessor.java Thu Apr 5 22:53:14 2012
@@ -74,42 +74,41 @@ public class ParserChunker2MatcherProces
private static final String MODEL_DIR_KEY = "nlp.models.dir";
// TODO config
// this is where resources should live
- private static String MODEL_DIR, MODEL_DIR_REL = "resources/models";
+ private static String MODEL_DIR, MODEL_DIR_REL = "src/test/resources/models111";
protected static ParserChunker2MatcherProcessor instance;
private SentenceDetector sentenceDetector;
private Tokenizer tokenizer;
private POSTagger posTagger;
- public SentenceDetector getSentenceDetector() {
- return sentenceDetector;
- }
-
- public void setSentenceDetector(SentenceDetector sentenceDetector) {
- this.sentenceDetector = sentenceDetector;
- }
-
- public Tokenizer getTokenizer() {
- return tokenizer;
- }
-
- public void setTokenizer(Tokenizer tokenizer) {
- this.tokenizer = tokenizer;
- }
-
- public ChunkerME getChunker() {
- return chunker;
- }
-
- public void setChunker(ChunkerME chunker) {
- this.chunker = chunker;
- }
-
private Parser parser;
private ChunkerME chunker;
private final int NUMBER_OF_SECTIONS_IN_SENTENCE_CHUNKS = 5;
private static Logger LOG = Logger.getLogger("opennlp.tools.textsimilarity.chunker2matcher.ParserChunker2MatcherProcessor");
private Map<String,String[][]> sentence_parseObject = new HashMap<String,String[][]>();
+ public SentenceDetector getSentenceDetector() {
+ return sentenceDetector;
+ }
+
+ public void setSentenceDetector(SentenceDetector sentenceDetector) {
+ this.sentenceDetector = sentenceDetector;
+ }
+
+ public Tokenizer getTokenizer() {
+ return tokenizer;
+ }
+
+ public void setTokenizer(Tokenizer tokenizer) {
+ this.tokenizer = tokenizer;
+ }
+
+ public ChunkerME getChunker() {
+ return chunker;
+ }
+
+ public void setChunker(ChunkerME chunker) {
+ this.chunker = chunker;
+ }
@SuppressWarnings("unchecked")
protected ParserChunker2MatcherProcessor() {
try {