You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by jo...@apache.org on 2017/02/01 15:32:13 UTC
opennlp git commit: OPENNLP-973: Respect setting of PrintMessages in
DataIndexer impls
Repository: opennlp
Updated Branches:
refs/heads/master 9d5b740c7 -> 284fe0ab9
OPENNLP-973: Respect setting of PrintMessages in DataIndexer impls
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/284fe0ab
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/284fe0ab
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/284fe0ab
Branch: refs/heads/master
Commit: 284fe0ab9d98c14a66aa32ada0c46b294f94329d
Parents: 9d5b740
Author: J�rn Kottmann <jo...@apache.org>
Authored: Wed Feb 1 15:38:19 2017 +0100
Committer: J�rn Kottmann <jo...@apache.org>
Committed: Wed Feb 1 16:21:22 2017 +0100
----------------------------------------------------------------------
.../tools/ml/model/AbstractDataIndexer.java | 2 +-
.../tools/ml/model/OnePassDataIndexer.java | 18 ++++++++---------
.../tools/ml/model/TwoPassDataIndexer.java | 21 ++++++++++----------
3 files changed, 21 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/284fe0ab/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractDataIndexer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractDataIndexer.java b/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractDataIndexer.java
index 0eabf87..be6a6e4 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractDataIndexer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractDataIndexer.java
@@ -133,7 +133,7 @@ public abstract class AbstractDataIndexer implements DataIndexer {
throw new InsufficientTrainingDataException("Insufficient training data to create model.");
}
- if (sort) System.out.println("done. Reduced " + numEvents + " events to " + numUniqueEvents + ".");
+ if (sort) display("done. Reduced " + numEvents + " events to " + numUniqueEvents + ".\n");
contexts = new int[numUniqueEvents][];
outcomeList = new int[numUniqueEvents];
http://git-wip-us.apache.org/repos/asf/opennlp/blob/284fe0ab/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassDataIndexer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassDataIndexer.java b/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassDataIndexer.java
index 7b53251..cf8fb7f 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassDataIndexer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassDataIndexer.java
@@ -48,24 +48,24 @@ public class OnePassDataIndexer extends AbstractDataIndexer {
List<Event> events;
List<ComparableEvent> eventsToCompare;
- System.out.println("Indexing events using cutoff of " + cutoff + "\n");
+ display("Indexing events using cutoff of " + cutoff + "\n\n");
- System.out.print("\tComputing event counts... ");
+ display("\tComputing event counts... ");
events = computeEventCounts(eventStream, predicateIndex, cutoff);
- System.out.println("done. " + events.size() + " events");
+ display("done. " + events.size() + " events\n");
- System.out.print("\tIndexing... ");
+ display("\tIndexing... ");
eventsToCompare = index(events, predicateIndex);
// done with event list
events = null;
// done with predicates
predicateIndex = null;
- System.out.println("done.");
+ display("done.\n");
- System.out.print("Sorting and merging events... ");
+ display("Sorting and merging events... ");
sortAndMerge(eventsToCompare, sort);
- System.out.println("Done indexing.");
+ display("Done indexing.\n");
}
/**
@@ -140,8 +140,8 @@ public class OnePassDataIndexer extends AbstractDataIndexer {
ce = new ComparableEvent(ocID, cons);
eventsToCompare.add(ce);
} else {
- System.err.println("Dropped event " + ev.getOutcome() + ":"
- + Arrays.asList(ev.getContext()));
+ display("Dropped event " + ev.getOutcome() + ":"
+ + Arrays.asList(ev.getContext()) + "\n");
}
// recycle the TIntArrayList
indexedContext.clear();
http://git-wip-us.apache.org/repos/asf/opennlp/blob/284fe0ab/opennlp-tools/src/main/java/opennlp/tools/ml/model/TwoPassDataIndexer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/model/TwoPassDataIndexer.java b/opennlp-tools/src/main/java/opennlp/tools/ml/model/TwoPassDataIndexer.java
index 133c350..b3cc89b 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/model/TwoPassDataIndexer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/model/TwoPassDataIndexer.java
@@ -55,17 +55,17 @@ public class TwoPassDataIndexer extends AbstractDataIndexer {
Map<String,Integer> predicateIndex = new HashMap<>();
List<ComparableEvent> eventsToCompare;
- System.out.println("Indexing events using cutoff of " + cutoff + "\n");
+ display("Indexing events using cutoff of " + cutoff + "\n\n");
- System.out.print("\tComputing event counts... ");
+ display("\tComputing event counts... ");
File tmp = File.createTempFile("events", null);
tmp.deleteOnExit();
Writer osw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(tmp),"UTF8"));
int numEvents = computeEventCounts(eventStream, osw, predicateIndex, cutoff);
- System.out.println("done. " + numEvents + " events");
+ display("done. " + numEvents + " events\n");
- System.out.print("\tIndexing... ");
+ display("\tIndexing... ");
try (FileEventStream fes = new FileEventStream(tmp)) {
eventsToCompare = index(numEvents, fes, predicateIndex);
@@ -73,16 +73,16 @@ public class TwoPassDataIndexer extends AbstractDataIndexer {
// done with predicates
predicateIndex = null;
tmp.delete();
- System.out.println("done.");
+ display("done.\n");
if (sort) {
- System.out.print("Sorting and merging events... ");
+ display("Sorting and merging events... ");
}
else {
- System.out.print("Collecting events... ");
+ display("Collecting events... ");
}
sortAndMerge(eventsToCompare,sort);
- System.out.println("Done indexing.");
+ display("Done indexing.\n");
}
/**
@@ -120,9 +120,11 @@ public class TwoPassDataIndexer extends AbstractDataIndexer {
return eventCount;
}
+ // TODO: merge this code with the copy and paste version in OnePassDataIndexer
private List<ComparableEvent> index(int numEvents, ObjectStream<Event> es,
Map<String,Integer> predicateIndex) throws IOException {
Map<String,Integer> omap = new HashMap<>();
+
int outcomeCount = 0;
List<ComparableEvent> eventsToCompare = new ArrayList<>(numEvents);
List<Integer> indexedContext = new ArrayList<>();
@@ -159,7 +161,7 @@ public class TwoPassDataIndexer extends AbstractDataIndexer {
eventsToCompare.add(ce);
}
else {
- System.err.println("Dropped event " + ev.getOutcome() + ":" + Arrays.asList(ev.getContext()));
+ display("Dropped event " + ev.getOutcome() + ":" + Arrays.asList(ev.getContext()) + "\n");
}
// recycle the TIntArrayList
indexedContext.clear();
@@ -168,6 +170,5 @@ public class TwoPassDataIndexer extends AbstractDataIndexer {
predLabels = toIndexedStringArray(predicateIndex);
return eventsToCompare;
}
-
}