You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by le...@apache.org on 2016/05/16 06:26:45 UTC
[29/66] [partial] incubator-joshua git commit: JOSHUA-252 Make it
possible to use Maven to build Joshua
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/8cdbc4b8/src/joshua/util/CommandLineParser.java
----------------------------------------------------------------------
diff --git a/src/joshua/util/CommandLineParser.java b/src/joshua/util/CommandLineParser.java
deleted file mode 100644
index d79fd55..0000000
--- a/src/joshua/util/CommandLineParser.java
+++ /dev/null
@@ -1,738 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package joshua.util;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * Java Command Line Parser
- * <p>
- * The current version supports string and integer options.
- * <p>
- * Support is not included for options which take a list of values.
- *
- * @author Lane O.B. Schwartz
- */
-@SuppressWarnings("rawtypes")
-public class CommandLineParser {
-
- private Map<Character, Option<Integer>> intShortForms;
- private Map<String, Option<Integer>> intLongForms;
-
- private Map<Character, Option<String>> stringShortForms;
- private Map<String, Option<String>> stringLongForms;
-
- private Map<Character, Option<Boolean>> booleanShortForms;
- private Map<String, Option<Boolean>> booleanLongForms;
-
- private List<Option> allOptions;
-
- private final Set<String> localizedTrueStrings = new HashSet<String>();
- private final Set<String> localizedFalseStrings = new HashSet<String>();
-
- public CommandLineParser() {
- intShortForms = new HashMap<Character, Option<Integer>>();
- intLongForms = new HashMap<String, Option<Integer>>();
-
- stringShortForms = new HashMap<Character, Option<String>>();
- stringLongForms = new HashMap<String, Option<String>>();
-
- booleanShortForms = new HashMap<Character, Option<Boolean>>();
- booleanLongForms = new HashMap<String, Option<Boolean>>();
-
- allOptions = new LinkedList<Option>();
-
- localizedTrueStrings.add("true");
- localizedTrueStrings.add("yes");
- localizedFalseStrings.add("false");
- localizedFalseStrings.add("no");
- }
-
- public CommandLineParser(Set<String> localizedTrueStrings, Set<String> localizedFalseStrings) {
- this();
-
- this.localizedTrueStrings.clear();
- this.localizedFalseStrings.clear();
-
- this.localizedTrueStrings.addAll(localizedTrueStrings);
- this.localizedFalseStrings.addAll(localizedFalseStrings);
- }
-
- public Option<Integer> addIntegerOption(char shortForm, String longForm, String valueVariable,
- Integer defaultValue, Set<Integer> legalValues, String comment) {
- if (shortForm != Option.MISSING_SHORT_FORM && (intShortForms.containsKey(shortForm))
- || (!longForm.equals(Option.MISSING_LONG_FORM) && intLongForms.containsKey(longForm)))
- throw new DuplicateOptionException("Duplicate options are not allowed");
-
- Option<Integer> o =
- new Option<Integer>(shortForm, longForm, valueVariable, defaultValue, legalValues, comment);
- intShortForms.put(shortForm, o);
- intLongForms.put(longForm, o);
- allOptions.add(o);
- return o;
- }
-
- public Option<Integer> addIntegerOption(char shortForm, String longForm, String valueVariable,
- Set<Integer> legalValues, String comment) {
- return addIntegerOption(shortForm, longForm, valueVariable, null, legalValues, comment);
- }
-
- public Option<Integer> addIntegerOption(char shortForm, String longForm, String valueVariable,
- String comment) {
- return addIntegerOption(shortForm, longForm, valueVariable, null, new UniversalSet<Integer>(),
- comment);
- }
-
- public Option<Integer> addIntegerOption(char shortForm, String longForm, String comment) {
- return addIntegerOption(shortForm, longForm, null, null, new UniversalSet<Integer>(), comment);
- }
-
- public Option<Integer> addIntegerOption(char shortForm, String longForm, String valueVariable,
- Integer defaultValue, String comment) {
- return addIntegerOption(shortForm, longForm, valueVariable, defaultValue,
- new UniversalSet<Integer>(), comment);
- }
-
- public Option<Integer> addIntegerOption(String longForm, String valueVariable,
- Integer defaultValue, String comment) {
- return addIntegerOption(Option.MISSING_SHORT_FORM, longForm, valueVariable, defaultValue,
- new UniversalSet<Integer>(), comment);
- }
-
- public Option<Integer> addIntegerOption(char shortForm, String longForm) {
- return addIntegerOption(shortForm, longForm, null, null, new UniversalSet<Integer>(), "");
- }
-
- public Option<Integer> addIntegerOption(char shortForm) {
- return addIntegerOption(shortForm, Option.MISSING_LONG_FORM);
- }
-
- public Option<Integer> addIntegerOption(String longForm) {
- return addIntegerOption(Option.MISSING_SHORT_FORM, longForm);
- }
-
- public Option<Integer> addIntegerOption(String longForm, String comment) {
- return addIntegerOption(Option.MISSING_SHORT_FORM, longForm, comment);
- }
-
-
- // String options
-
-
- public Option<String> addStringOption(char shortForm, String longForm, String valueVariable,
- String defaultValue, Set<String> legalValues, String comment) {
- if (shortForm != Option.MISSING_SHORT_FORM && (intShortForms.containsKey(shortForm))
- || (!longForm.equals(Option.MISSING_LONG_FORM) && intLongForms.containsKey(longForm)))
- throw new DuplicateOptionException("Duplicate options are not allowed");
-
- Option<String> o =
- new Option<String>(shortForm, longForm, valueVariable, defaultValue, legalValues, comment);
- stringShortForms.put(shortForm, o);
- stringLongForms.put(longForm, o);
- allOptions.add(o);
- return o;
- }
-
- public Option<String> addStringOption(char shortForm, String longForm, String valueVariable,
- Set<String> legalValues, String comment) {
- return addStringOption(shortForm, longForm, valueVariable, null, legalValues, comment);
- }
-
- public Option<String> addStringOption(char shortForm, String longForm, String valueVariable,
- String comment) {
- return addStringOption(shortForm, longForm, valueVariable, null, new UniversalSet<String>(),
- comment);
- }
-
- public Option<String> addStringOption(String longForm, String valueVariable, String comment) {
- return addStringOption(Option.MISSING_SHORT_FORM, longForm, valueVariable, null,
- new UniversalSet<String>(), comment);
- }
-
- public Option<String> addStringOption(char shortForm, String longForm, String comment) {
- return addStringOption(shortForm, longForm, null, null, new UniversalSet<String>(), comment);
- }
-
- public Option<String> addStringOption(char shortForm, String longForm, String valueVariable,
- String defaultValue, String comment) {
- return addStringOption(shortForm, longForm, valueVariable, defaultValue,
- new UniversalSet<String>(), comment);
- }
-
- public Option<String> addStringOption(String longForm, String valueVariable, String defaultValue,
- String comment) {
- return addStringOption(Option.MISSING_SHORT_FORM, longForm, valueVariable, defaultValue,
- new UniversalSet<String>(), comment);
- }
-
- public Option<String> addStringOption(char shortForm, String longForm) {
- return addStringOption(shortForm, longForm, null, null, new UniversalSet<String>(), "");
- }
-
- public Option<String> addStringOption(char shortForm) {
- return addStringOption(shortForm, Option.MISSING_LONG_FORM);
- }
-
- public Option<String> addStringOption(String longForm) {
- return addStringOption(Option.MISSING_SHORT_FORM, longForm);
- }
-
- public Option<String> addStringOption(String longForm, String comment) {
- return addStringOption(Option.MISSING_SHORT_FORM, longForm, comment);
- }
-
-
- // boolean options
-
- public Option<Boolean> addBooleanOption(char shortForm, String longForm, String valueVariable,
- Boolean defaultValue, String comment) {
- if (shortForm != Option.MISSING_SHORT_FORM && (booleanShortForms.containsKey(shortForm))
- || (!longForm.equals(Option.MISSING_LONG_FORM) && booleanLongForms.containsKey(longForm)))
- throw new DuplicateOptionException("Duplicate options are not allowed");
- Set<Boolean> legalBooleanValues = new HashSet<Boolean>();
- legalBooleanValues.add(true);
- legalBooleanValues.add(false);
-
- Option<Boolean> o =
- new Option<Boolean>(shortForm, longForm, valueVariable, defaultValue, legalBooleanValues,
- comment);
- booleanShortForms.put(shortForm, o);
- booleanLongForms.put(longForm, o);
- allOptions.add(o);
- return o;
- }
-
- public Option<Boolean> addBooleanOption(char shortForm, String longForm, String valueVariable,
- String comment) {
- return addBooleanOption(shortForm, longForm, valueVariable, null, comment);
- }
-
- public Option<Boolean> addBooleanOption(char shortForm, String longForm, String comment) {
- return addBooleanOption(shortForm, longForm, null, null, comment);
- }
-
- public Option<Boolean> addBooleanOption(String longForm, Boolean defaultValue, String comment) {
- return addBooleanOption(Option.MISSING_SHORT_FORM, longForm, null, defaultValue, comment);
- }
-
- public Option<Boolean> addBooleanOption(String longForm, String valueVariable,
- Boolean defaultValue, String comment) {
- return addBooleanOption(Option.MISSING_SHORT_FORM, longForm, valueVariable, defaultValue,
- comment);
- }
-
- public Option<Boolean> addBooleanOption(char shortForm, String longForm) {
- return addBooleanOption(shortForm, longForm, null, null, "");
- }
-
- public Option<Boolean> addBooleanOption(char shortForm) {
- return addBooleanOption(shortForm, Option.MISSING_LONG_FORM);
- }
-
- public Option<Boolean> addBooleanOption(String longForm) {
- return addBooleanOption(Option.MISSING_SHORT_FORM, longForm);
- }
-
- public Option<Boolean> addBooleanOption(String longForm, String comment) {
- return addBooleanOption(Option.MISSING_SHORT_FORM, longForm, comment);
- }
-
-
-
- // float options
-
-
-
- // /
- /*
- * public Option<Integer> addIntegerOption(char shortForm, String longForm) { if
- * (intShortForms.containsKey(shortForm) || intLongForms.containsKey(longForm)) throw new
- * DuplicateOptionException("Duplicate options are not allowed");
- *
- * Option<Integer> o = new Option<Integer>(shortForm, longForm); intShortForms.put(shortForm, o);
- * intLongForms.put(longForm, o); allOptions.add(o);
- *
- * return o; }
- *
- * public Option<Integer> addIntegerOption(char shortForm, String longForm, String valueVariable,
- * int defaultValue, Set<Integer> legalValues, String comment) { if
- * (intShortForms.containsKey(shortForm) || intLongForms.containsKey(longForm)) throw new
- * DuplicateOptionException("Duplicate options are not allowed");
- *
- * Option<Integer> o = new Option<Integer>(shortForm, longForm, valueVariable, defaultValue,
- * comment); intShortForms.put(shortForm, o); intLongForms.put(longForm, o); allOptions.add(o);
- * return o; }
- *
- * public Option<Integer> addIntegerOption(char shortForm, String longForm, String valueVariable,
- * int defaultValue, String comment) { if (intShortForms.containsKey(shortForm) ||
- * intLongForms.containsKey(longForm)) throw new
- * DuplicateOptionException("Duplicate options are not allowed");
- *
- * Option<Integer> o = new Option<Integer>(shortForm, longForm, valueVariable, defaultValue,
- * comment); intShortForms.put(shortForm, o); intLongForms.put(longForm, o); allOptions.add(o);
- * return o; }
- *
- * public Option<Integer> addIntegerOption(char shortForm, String longForm, String valueVariable,
- * String comment) { if (intShortForms.containsKey(shortForm) ||
- * intLongForms.containsKey(longForm)) throw new
- * DuplicateOptionException("Duplicate options are not allowed");
- *
- * Option<Integer> o = new Option<Integer>(shortForm, longForm, valueVariable, comment);
- * intShortForms.put(shortForm, o); intLongForms.put(longForm, o); allOptions.add(o); return o; }
- */
-
- /*
- * public Option<String> addStringOption(char shortForm, String longForm) { if
- * (stringShortForms.containsKey(shortForm) || stringLongForms.containsKey(longForm)) throw new
- * DuplicateOptionException("Duplicate options are not allowed");
- *
- * Option<String> o = new Option<String>(shortForm, longForm); stringShortForms.put(shortForm, o);
- * stringLongForms.put(longForm, o); allOptions.add(o); return o; }
- */
-
- public void parse(String[] argv) {
-
- Collection<Option> parsedOptions = new HashSet<Option>();
-
- int index = 0;
-
- while (index < argv.length) {
- if (argv[index].startsWith("--")) {
- int splitPoint = argv[index].indexOf('=');
- if (splitPoint == 2) {
- throw new CommandLineParserException("Invalid option: --");
- } else if (splitPoint >= 0) {
- String option = argv[index].substring(2, splitPoint);
- String value = argv[index].substring(splitPoint + 1);
- parsedOptions.add(parseLongForm(option, value));
- } else if (index + 1 < argv.length) {
- String option = argv[index].substring(2);
- String value = argv[index + 1];
- if (value.startsWith("-") && !value.equals("-") && !value.equals("--")) {
- parsedOptions.add(parseLongForm(option));
- } else {
- parsedOptions.add(parseLongForm(option, value));
- index++;
- }
- } else {
- // Must be a boolean option
- String option = argv[index].substring(2);
- parsedOptions.add(parseLongForm(option));
- // throw new CommandLineParserException("No value provided for option " +
- // argv[index].substring(2));
- }
- } else if (argv[index].startsWith("-")) {
- String option = argv[index].substring(1);
- if (option.length() == 1) {
- if (index + 1 < argv.length) {
- String value = argv[index + 1];
- if (value.startsWith("-") && !value.equals("-") && !value.equals("--")) {
- // Must be a boolean option
- parsedOptions.add(parseShortForm(option.charAt(0)));
- } else {
- parsedOptions.add(parseShortForm(option.charAt(0), value));
- index++;
- }
- } else {
- // Must be a boolean option
- parsedOptions.add(parseShortForm(option.charAt(0)));
- }
- } else {
- throw new CommandLineParserException(argv[index] + " is not a valid option");
- }
- }
- index++;
- }
-
- for (Option o : allOptions) {
- if (o.isRequired() && !parsedOptions.contains(o)) {
- die("A required option was not provided:\n " + o + "\n");
- }
- }
-
- }
-
- public void printUsage() {
- System.err.println("Usage:");
- for (Option o : allOptions) {
- System.err.println(o);
- }
- }
-
- private void die(String error) {
- System.err.println(error);
- printUsage();
- System.exit(1);
- }
-
- public Option parseLongForm(String key, String value) {
-
- if (intLongForms.containsKey(key)) {
- try {
- Option<Integer> o = intLongForms.get(key);
- o.setValue(Integer.valueOf(value));
- return o;
- } catch (NumberFormatException e) {
- die("Option " + key + " requires an integer value.");
- return null;
- }
- } else if (stringLongForms.containsKey(key)) {
- Option<String> o = stringLongForms.get(key);
- o.setValue(value);
- return o;
- } else if (booleanLongForms.containsKey(key)) {
- Option<Boolean> o = booleanLongForms.get(key);
-
- if (localizedTrueStrings.contains(value.toLowerCase())) {
- o.setValue(true);
- } else if (localizedFalseStrings.contains(value.toLowerCase())) {
- o.setValue(false);
- } else {
- throw new CommandLineParserException("Invalid value \"" + value + "\" for boolean option "
- + key);
- }
-
- return o;
- } else {
-
- throw new Error("Bug in command line parser - unexpected option type encountered for option "
- + key);
- }
- }
-
- public Option parseLongForm(String key) {
-
- if (booleanLongForms.containsKey(key)) {
- Option<Boolean> o = booleanLongForms.get(key);
- o.setValue(true);
- return o;
-
- } else {
- throw new CommandLineParserException("No such boolean option exists: --" + key);
- }
- }
-
- public Option parseShortForm(Character key) {
-
- if (booleanShortForms.containsKey(key)) {
- Option<Boolean> o = booleanShortForms.get(key);
- o.setValue(true);
- return o;
-
- } else {
- throw new CommandLineParserException("No such boolean option exists: -" + key);
- }
- }
-
- public Option parseShortForm(Character key, String value) {
- if (intShortForms.containsKey(key)) {
- try {
- Option<Integer> o = intShortForms.get(key);
- o.setValue(Integer.valueOf(value));
- return o;
- } catch (NumberFormatException e) {
- die("Option " + key + " requires an integer value.");
- return null;
- }
- } else if (stringShortForms.containsKey(key)) {
- Option<String> o = stringShortForms.get(key);
- o.setValue(value);
- return o;
- } else if (booleanShortForms.containsKey(key)) {
- Option<Boolean> o = booleanShortForms.get(key);
-
- if (localizedTrueStrings.contains(value.toLowerCase())) {
- o.setValue(true);
- } else if (localizedFalseStrings.contains(value.toLowerCase())) {
- o.setValue(false);
- } else {
- throw new CommandLineParserException("Invalid value \"" + value + "\" for boolean option "
- + key);
- }
-
- return o;
- } else {
- throw new Error("Bug in command line parser - unexpected option type encountered");
- }
- }
-
- /*
- * public int intValue(Option o) { if (intOptions.containsKey(o)) return intOptions.get(o); else
- * throw new RuntimeException("No such integer option"); }
- *
- * public String stringValue(Option o) { if (stringOptions.containsKey(o)) return
- * stringOptions.get(o); else throw new RuntimeException("No such string option"); }
- */
-
- public <OptionType> OptionType getValue(Option<OptionType> option) {
- return option.getValue();
- }
-
- public boolean hasValue(Option<?> option) {
- return option.hasValue();
- }
-
- public static void main(String[] args) {
- CommandLineParser parser = new CommandLineParser();
- Option<Integer> n = parser.addIntegerOption('n', "number", "NUMBER", "a number to be supplied");
-
- parser.parse(args);
-
- // parser.printUsage();
- System.out.println(parser.getValue(n));
- }
-
- @SuppressWarnings("serial")
- public static class CommandLineParserException extends RuntimeException {
- public CommandLineParserException(String message) {
- super(message);
- }
- }
-
- @SuppressWarnings("serial")
- public static class DuplicateOptionException extends RuntimeException {
- public DuplicateOptionException(String message) {
- super(message);
- }
- }
-
- public class Option<OptionType> {
- private final char shortForm;
- private final String longForm;
- private final String comment;
- private final OptionType defaultValue;
- private final String valueVariable;
- private final Set<OptionType> legalValues;
-
- public static final char MISSING_SHORT_FORM = '\u0000';
- public static final String MISSING_LONG_FORM = "\u0000";
-
- private OptionType optionValue;
-
- public Option(char shortForm, String longForm, String valueVariable, OptionType defaultValue,
- Set<OptionType> legalValues, String comment) {
-
- if (longForm == null) throw new NullPointerException("longForm must not be null");
-
- if (comment == null) throw new NullPointerException("comment must not be null");
-
- this.shortForm = shortForm;
- this.longForm = longForm;
- this.comment = comment;
- this.valueVariable = valueVariable;
- this.defaultValue = defaultValue;
- this.legalValues = legalValues;
- this.optionValue = null;
- }
-
- public Option(char shortForm, String longForm, String valueVariable,
- Set<OptionType> legalValues, String comment) {
- this(shortForm, longForm, valueVariable, null, legalValues, comment);
- }
-
-
- public Option(char shortForm, String longForm, String valueVariable, String comment) {
- this(shortForm, longForm, valueVariable, null, new UniversalSet<OptionType>(), comment);
- }
-
- public Option(char shortForm, String longForm, String comment) {
- this(shortForm, longForm, null, null, new UniversalSet<OptionType>(), comment);
- }
-
- public Option(char shortForm, String longForm, String valueVariable, OptionType defaultValue,
- String comment) {
- this(shortForm, longForm, valueVariable, defaultValue, new UniversalSet<OptionType>(),
- comment);
- }
-
- public Option(String longForm, String valueVariable, OptionType defaultValue, String comment) {
- this(MISSING_SHORT_FORM, longForm, valueVariable, defaultValue,
- new UniversalSet<OptionType>(), comment);
- }
-
- public Option(char shortForm, String longForm) {
- this(shortForm, longForm, null, null, new UniversalSet<OptionType>(), "");
- }
-
- public Option(char shortForm) {
- this(shortForm, MISSING_LONG_FORM);
- }
-
- public Option(String longForm) {
- this(MISSING_SHORT_FORM, longForm);
- }
-
- public Option(String longForm, String comment) {
- this(MISSING_SHORT_FORM, longForm, comment);
- }
-
- public boolean isOptional() {
- return (null != defaultValue);
- }
-
- public boolean isRequired() {
- return (null == defaultValue);
- }
-
- public char getShortForm() {
- return shortForm;
- }
-
- public String getLongForm() {
- return longForm;
- }
-
- public String getComment() {
- return comment;
- }
-
- void setValue(OptionType value) {
- this.optionValue = value;
- }
-
- OptionType getValue() {
- if (optionValue != null) {
- return optionValue;
- } else if (defaultValue != null) {
- return defaultValue;
- } else {
- throw new CommandLineParserException(
- "Unable to get value because option has not been initialized and does not have a default value: "
- + this.toString());
- }
- }
-
- boolean hasValue() {
- return !(null == optionValue && null == defaultValue);
- }
-
- public String toString() {
-
- String formattedShortForm;
- if (shortForm == Option.MISSING_SHORT_FORM) {
- formattedShortForm = "";
- } else {
- formattedShortForm = "-" + shortForm;
- }
-
- String formattedLongForm;
- if (longForm.equals(Option.MISSING_LONG_FORM)) {
- formattedLongForm = "";
- } else {
- formattedLongForm = "--" + longForm;
- }
-
- if (shortForm != Option.MISSING_SHORT_FORM && !longForm.equals(Option.MISSING_LONG_FORM)) {
- formattedShortForm += ",";
- }
-
- if (valueVariable != null && valueVariable.length() >= 1) {
- formattedLongForm += "=" + valueVariable;
- }
-
- String string = String.format(" %1$3s %2$-21s", formattedShortForm, formattedLongForm);
-
- if (null != comment) {
- string += " " + comment;
- }
-
- if (!(legalValues instanceof UniversalSet)) {
- string += " " + legalValues;
- }
-
- return string;
- }
-
- public boolean equals(Object o) {
- if (o instanceof Option) {
- return (shortForm == ((Option) o).shortForm && longForm == ((Option) o).longForm);
- } else {
- return false;
- }
- }
-
- public int hashCode() {
- return (shortForm + longForm).hashCode();
- }
- }
-
- static class UniversalSet<E> implements Set<E> {
-
- public boolean add(Object o) {
- throw new UnsupportedOperationException();
- }
-
- public boolean addAll(Collection c) {
- throw new UnsupportedOperationException();
- }
-
- public void clear() {
- throw new UnsupportedOperationException();
- }
-
- public boolean contains(Object o) {
- return true;
- }
-
- public boolean containsAll(Collection c) {
- return true;
- }
-
- public boolean isEmpty() {
- return false;
- }
-
- public Iterator<E> iterator() {
- return null;
- }
-
- public boolean remove(Object o) {
- throw new UnsupportedOperationException();
- }
-
- public boolean removeAll(Collection c) {
- throw new UnsupportedOperationException();
- }
-
- public boolean retainAll(Collection c) {
- throw new UnsupportedOperationException();
- }
-
- public int size() {
- return Integer.MAX_VALUE;
- }
-
- public Object[] toArray() {
- return null;
- }
-
- public <T> T[] toArray(T[] a) {
- return null;
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/8cdbc4b8/src/joshua/util/CompareGrammars.java
----------------------------------------------------------------------
diff --git a/src/joshua/util/CompareGrammars.java b/src/joshua/util/CompareGrammars.java
deleted file mode 100644
index 109d7a1..0000000
--- a/src/joshua/util/CompareGrammars.java
+++ /dev/null
@@ -1,207 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package joshua.util;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.util.HashSet;
-import java.util.Scanner;
-import java.util.Set;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import joshua.decoder.ff.tm.format.HieroFormatReader;
-
-/**
- * This class allows two grammars (loaded from disk) to be compared.
- *
- * @author Lane Schwartz
- */
-public class CompareGrammars {
-
- /** Logger for this class. */
- private static final Logger logger = Logger.getLogger(CompareGrammars.class.getName());
-
- /**
- * Gets a set containing all unique instances of the specified field.
- *
- * @param grammarFile File containing a grammar.
- * @param fieldDelimiter Regular expression to split each line
- * @param fieldNumber Field from each rule to extract
- * @return set containing all unique instances of the specified field
- * @throws FileNotFoundException
- */
- public static Set<String> getFields(File grammarFile, String fieldDelimiter, int fieldNumber)
- throws FileNotFoundException {
-
- Scanner grammarScanner = new Scanner(grammarFile);
-
- Set<String> set = new HashSet<String>();
-
- while (grammarScanner.hasNextLine()) {
-
- String line = grammarScanner.nextLine();
-
- String[] fields = line.split(fieldDelimiter);
-
- set.add(fields[fieldNumber]);
- }
-
- grammarScanner.close();
-
- return set;
- }
-
- public static void compareValues(File grammarFile1, File grammarFile2, String fieldDelimiter,
- int fieldNumber, String scoresDelimiter, int scoresFieldNumber, float delta)
- throws FileNotFoundException {
-
- Scanner grammarScanner1 = new Scanner(grammarFile1);
- Scanner grammarScanner2 = new Scanner(grammarFile2);
-
- Set<String> set = new HashSet<String>();
-
- int counter = 0;
- float totalOverDiffs = 0.0f;
- while (grammarScanner1.hasNextLine() && grammarScanner2.hasNextLine()) {
-
- counter++;
-
- String line1 = grammarScanner1.nextLine();
- String[] fields1 = line1.split(fieldDelimiter);
- String[] scores1 = fields1[fieldNumber].split(scoresDelimiter);
- float score1 = Float.valueOf(scores1[scoresFieldNumber]);
-
- String line2 = grammarScanner2.nextLine();
- String[] fields2 = line2.split(fieldDelimiter);
- String[] scores2 = fields2[fieldNumber].split(scoresDelimiter);
- float score2 = Float.valueOf(scores2[scoresFieldNumber]);
-
- if (fields1[0].endsWith(fields2[0]) && fields1[1].endsWith(fields2[1])
- && fields1[1].endsWith(fields2[1])) {
-
- float diff1 = Math.abs(score1 - score2);
- float diff2 = Math.abs(score2 - score1);
- float diff = (diff1 < diff2) ? diff1 : diff2;
-
- if (diff > delta) {
- logger.fine("Line " + counter + ": Score mismatch: " + score1 + " vs " + score2);
- set.add(line1);
- totalOverDiffs += diff;
- } else if (logger.isLoggable(Level.FINEST)) {
- logger.finest("Line " + counter + ": Scores MATCH: " + score1 + " vs " + score2);
- }
-
- } else {
- throw new RuntimeException("Lines don't match: " + line1 + " and " + line2);
- }
- }
-
- grammarScanner1.close();
- grammarScanner2.close();
-
- if (set.isEmpty()) {
- logger.info("No score mismatches");
- } else {
- logger.warning("Number of mismatches: " + set.size() + " out of " + counter);
- logger.warning("Total mismatch logProb mass: " + totalOverDiffs + " (" + totalOverDiffs
- / set.size() + ") (" + totalOverDiffs / counter + ")");
- }
- }
-
- /**
- * Main method.
- *
- * @param args names of the two grammars to be compared
- * @throws FileNotFoundException
- */
- public static void main(String[] args) throws FileNotFoundException {
-
- if (args.length != 2) {
- logger.severe("Usage: " + CompareGrammars.class.toString() + " grammarFile1 grammarFile2");
- System.exit(-1);
- }
-
- // Tell standard in and out to use UTF-8
- FormatUtils.useUTF8();
- logger.finest("Using UTF-8");
-
- logger.info("Comparing grammar files " + args[0] + " and " + args[1]);
-
- File grammarFile1 = new File(args[0]);
- File grammarFile2 = new File(args[1]);
-
- String fieldDelimiter = HieroFormatReader.getFieldDelimiter();
-
- boolean compareScores = true;
-
- // Compare left-hand sides
- {
- Set<String> leftHandSides1 = getFields(grammarFile1, fieldDelimiter, 0);
- Set<String> leftHandSides2 = getFields(grammarFile2, fieldDelimiter, 0);
-
- if (leftHandSides1.equals(leftHandSides2)) {
- logger.info("Grammar files have the same set of left-hand sides");
- } else {
- logger.warning("Grammar files have differing sets of left-hand sides");
- compareScores = false;
- }
- }
-
- // Compare source right-hand sides
- {
- Set<String> sourceRHSs1 = getFields(grammarFile1, fieldDelimiter, 1);
- Set<String> sourceRHSs2 = getFields(grammarFile2, fieldDelimiter, 1);
-
- if (sourceRHSs1.equals(sourceRHSs2)) {
- logger.info("Grammar files have the same set of source right-hand sides");
- } else {
- logger.warning("Grammar files have differing sets of source right-hand sides");
- compareScores = false;
- }
- }
-
-
- // Compare target right-hand sides
- {
- Set<String> targetRHSs1 = getFields(grammarFile1, fieldDelimiter, 2);
- Set<String> targetRHSs2 = getFields(grammarFile2, fieldDelimiter, 2);
-
- if (targetRHSs1.equals(targetRHSs2)) {
- logger.info("Grammar files have the same set of target right-hand sides");
- } else {
- logger.warning("Grammar files have differing sets of target right-hand sides");
- compareScores = false;
- }
- }
-
- // Compare translation probs
- if (compareScores) {
- float delta = 0.001f;
- compareValues(grammarFile1, grammarFile2, fieldDelimiter, 3, "\\s+", 0, delta);
- compareValues(grammarFile1, grammarFile2, fieldDelimiter, 3, "\\s+", 1, delta);
- compareValues(grammarFile1, grammarFile2, fieldDelimiter, 3, "\\s+", 2, delta);
-
- }
-
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/8cdbc4b8/src/joshua/util/Counted.java
----------------------------------------------------------------------
diff --git a/src/joshua/util/Counted.java b/src/joshua/util/Counted.java
deleted file mode 100644
index 1014e12..0000000
--- a/src/joshua/util/Counted.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package joshua.util;
-
-import java.util.Comparator;
-
-/**
- * Represents an object being counted, with the associated count.
- *
- * @author Lane Schwartz
- */
-public class Counted<E> implements Comparable<Counted<E>> {
-
- /** The element being counted. */
- private final E element;
-
- /** The count associated with the element. */
- private final Integer count;
-
- /**
- * Constructs an object wrapping an element and its associated count.
- *
- * @param element An element being counted
- * @param count The count associated with the element
- */
- public Counted(E element, int count) {
- this.element = element;
- this.count = count;
- }
-
- /**
- * Gets the count associated with this object's element.
- *
- * @return The count associated with this object's element
- */
- public int getCount() {
- return count;
- }
-
- /**
- * Gets the element associated with this object.
- *
- * @return The element associated with this object
- */
- public E getElement() {
- return element;
- }
-
- /**
- * Compares this object to another counted object, according to the natural order of the counts
- * associated with each object.
- *
- * @param o Another counted object
- * @return -1 if the count of this object is less than the count of the other object, 0 if the
- * counts are equal, or 1 if the count of this object is greater than the count of the
- * other object
- */
- public int compareTo(Counted<E> o) {
- return count.compareTo(o.count);
- }
-
- /**
- * Gets a comparator that compares two counted objects based on the reverse of the natural order
- * of the counts associated with each object.
- *
- * @param <E>
- * @return A comparator that compares two counted objects based on the reverse of the natural
- * order of the counts associated with each object
- */
- public static <E> Comparator<Counted<E>> getDescendingComparator() {
- return new Comparator<Counted<E>>() {
- public int compare(Counted<E> o1, Counted<E> o2) {
- return (o2.count.compareTo(o1.count));
- }
- };
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/8cdbc4b8/src/joshua/util/Counts.java
----------------------------------------------------------------------
diff --git a/src/joshua/util/Counts.java b/src/joshua/util/Counts.java
deleted file mode 100644
index 4a20009..0000000
--- a/src/joshua/util/Counts.java
+++ /dev/null
@@ -1,308 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package joshua.util;
-
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Map.Entry;
-
-/**
- * Maintains element co-occurrence data.
- *
- * @author Lane Schwartz
- * @author Chris Callison-Burch
- * @param <A>
- * @param <B>
- */
-public class Counts<A, B> implements Iterable<Pair<A, B>> {
-
- /**
- * Stores the number of times instances of A and B co-occur.
- */
- private Map<A, Map<B, Integer>> counts;
-
- /** Stores the number of times instances of B occur. */
- private Map<B, Integer> bTotals;
-
- /** Stores relative frequency estimates for p(A | B). */
- private Map<A, Map<B, Float>> probabilities;
-
- /** Stores relative frequency estimates for p(B | A). */
- private Map<B, Map<A, Float>> reverseProbabilities;
-
- /** Stores the value to return when an unseen pair is queried. */
- private float floorProbability;
-
- /**
- * Constructs an initially empty co-occurrence counter, with floor probability set to
- * <code>Float.MIN_VALUE</code>.
- */
- public Counts() {
- this(Float.MIN_VALUE);
- }
-
- /**
- * Constructs an initially empty co-occurrence counter.
- *
- * @param floorProbability Floor probability to use when an unseen pair is queried.
- */
- public Counts(float floorProbability) {
- this.floorProbability = floorProbability;
- this.counts = new HashMap<A, Map<B, Integer>>();
- this.bTotals = new HashMap<B, Integer>();
- this.probabilities = new HashMap<A, Map<B, Float>>();
- this.reverseProbabilities = new HashMap<B, Map<A, Float>>();
- }
-
-
- /**
- * Increments the co-occurrence count of the provided objects.
- *
- * @param a
- * @param b
- */
- public void incrementCount(A a, B b) {
- // increment the count and handle the adding of objects to the map if they aren't already there
- {
- Map<B, Integer> bMap;
- if (counts.containsKey(a)) {
- bMap = counts.get(a);
- } else {
- bMap = new HashMap<B, Integer>();
- counts.put(a, bMap);
- }
-
- Integer previousCount;
- if (bMap.containsKey(b)) {
- previousCount = bMap.get(b);
- } else {
- previousCount = 0;
- }
- bMap.put(b, previousCount + 1);
- }
-
- // increments total for o2.
- {
- Integer previousTotal;
- if (bTotals.containsKey(b)) {
- previousTotal = bTotals.get(b);
- } else {
- previousTotal = 0;
- }
- bTotals.put(b, previousTotal + 1);
- }
-
- // Invalidate previously calculated probabilities
- {
- if (probabilities.containsKey(a)) {
- probabilities.get(a).clear();
- }
-
- if (reverseProbabilities.containsKey(b)) {
- reverseProbabilities.get(b).clear();
- }
- }
- }
-
- /**
- * Gets the co-occurrence count for the two elements.
- *
- * @param a
- * @param b
- * @return the co-occurrence count for the two elements
- */
- public int getCount(A a, B b) {
-
- int count = 0;
- if (counts.containsKey(a)) {
- Map<B, Integer> bMap = counts.get(a);
- if (bMap.containsKey(b)) {
- count = bMap.get(b);
- }
- }
-
- return count;
- }
-
- /**
- * Gets the total number of times the specified element has been seen.
- *
- * @param b
- * @return the total number of times the specified element has been seen
- */
- int getCount(B b) {
-
- return (bTotals.containsKey(b) ? bTotals.get(b) : 0);
-
- }
-
- /**
- * Gets the probability of a given b.
- * <p>
- * This value is the relative frequency estimate.
- *
- * @param a
- * @param b
- * @return the probability of a given b.
- */
- public float getProbability(A a, B b) {
-
- int count = getCount(a, b);
- int bCount = getCount(b);
-
- Float value;
- if (count == 0 || bCount == 0) {
-
- value = floorProbability;
-
- } else {
-
- Map<B, Float> bMap;
- if (probabilities.containsKey(a)) {
- bMap = probabilities.get(a);
- } else {
- bMap = new HashMap<B, Float>();
- }
-
-
- if (bMap.containsKey(b)) {
- value = bMap.get(b);
- } else {
- value = (float) count / (float) getCount(b);
- bMap.put(b, value);
- }
-
- }
-
- return value;
- }
-
- /**
- * Gets the probability of b given a.
- * <p>
- * This value is the relative frequency estimate in the reverse direction.
- *
- * @param b
- * @param a
- * @return the probability of b given a.
- */
- public float getReverseProbability(B b, A a) {
-
- int count = getCount(a, b);
-
- Float value = floorProbability;
-
- if (count > 0) {
-
- int aCount = 0;
- for (Integer aValue : counts.get(a).values()) {
- aCount += aValue;
- }
-
- if (aCount > 0) {
-
- Map<A, Float> aMap;
- if (reverseProbabilities.containsKey(b)) {
- aMap = reverseProbabilities.get(b);
- } else {
- aMap = new HashMap<A, Float>();
- }
-
- if (aMap.containsKey(a)) {
- value = aMap.get(a);
- } else {
- value = (float) count / (float) aCount;
- }
-
- }
-
- }
-
- return value;
-
- }
-
- /**
- * Gets the floor probability that is returned whenever an unseen pair is queried.
- *
- * @return The floor probability that is returned whenever an unseen pair is queried
- */
- public float getFloorProbability() {
- return this.floorProbability;
- }
-
- public void writeExternal(ObjectOutput out) throws IOException {
- out.writeObject(counts);
- out.writeObject(bTotals);
- out.writeObject(probabilities);
- out.writeObject(reverseProbabilities);
- out.writeFloat(floorProbability);
- // out.close();
- }
-
- @SuppressWarnings("unchecked")
- public void readExternal(ObjectInput in) throws ClassNotFoundException, IOException {
- this.counts = (HashMap<A, Map<B, Integer>>) in.readObject();
- this.bTotals = (HashMap<B, Integer>) in.readObject();
- this.probabilities = (HashMap<A, Map<B, Float>>) in.readObject();
- this.reverseProbabilities = (HashMap<B, Map<A, Float>>) in.readObject();
- this.floorProbability = in.readFloat();
- }
-
- /**
- * Gets an iterator over all counted pairs.
- * <p>
- * The pairs are not guaranteed to be iterated over in any particular order.
- *
- * @return an iterator over all counted pairs
- */
- public Iterator<Pair<A, B>> iterator() {
-
- final Iterator<Entry<A, Map<B, Integer>>> aIterator = counts.entrySet().iterator();
-
- return new Iterator<Pair<A, B>>() {
-
- Entry<A, Map<B, Integer>> entry = null;
- Iterator<B> bIterator = null;
-
- public boolean hasNext() {
- return (bIterator != null && bIterator.hasNext()) || aIterator.hasNext();
- }
-
- public Pair<A, B> next() {
- if (bIterator == null || !bIterator.hasNext()) {
- entry = aIterator.next();
- bIterator = entry.getValue().keySet().iterator();
- }
-
- return new Pair<A, B>(entry.getKey(), bIterator.next());
- }
-
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- };
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/8cdbc4b8/src/joshua/util/ExtractTopCand.java
----------------------------------------------------------------------
diff --git a/src/joshua/util/ExtractTopCand.java b/src/joshua/util/ExtractTopCand.java
deleted file mode 100644
index c24f970..0000000
--- a/src/joshua/util/ExtractTopCand.java
+++ /dev/null
@@ -1,179 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package joshua.util;
-
-import java.io.BufferedWriter;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-
-import joshua.util.io.IndexedReader;
-import joshua.util.io.LineReader;
-
-
-/**
- * This program extracts the 1-best output translations from the n-best output translations
- * generated by {@link joshua.decoder.Decoder}.
- *
- * @author wren ng thornton <wr...@users.sourceforge.net>
- * @version $LastChangedDate: 2009-03-26 15:06:57 -0400 (Thu, 26 Mar 2009) $
- */
-/*
- * TODO: This class should be renamed, something like ExtractBestCandidates or
- * ExtractBestTranslations. Saying "top" implies more than one (the top how many?) and "cand" is
- * unnecessary abbreviation (also, who cares about candidacy?). Once we rename this, the
- * ./example2/decode_example2.sh script will need updating (as will the end-to-end code)
- */
-public class ExtractTopCand {
-
- /**
- * Usage: <code>java ExtractTopCand nbestInputFile 1bestOutputFile</code>.
- * <p>
- * If the input file name is "-" then input is read from <code>System.in</code>. If the output
- * file name is "-" then output is directed to <code>System.out</code>. If a file already exists
- * with the output file name, it is truncated before writing. The bulk of this program is
- * implemented by {@link #extractOneBest(IndexedReader,BufferedWriter)}.
- */
- public static void main(String[] args) {
- String inFile = "-";
- String outFile = "-";
- int field = 1;
- if (args.length == 1) {
- inFile = args[0];
- } else if (args.length == 2) {
- inFile = args[0];
- outFile = args[1];
- } else if (args.length == 3) {
- inFile = args[0];
- outFile = args[1];
- field = Integer.parseInt(args[2]);
- } else {
- System.err
- .println("Usage: ExtractTopCand [nbestInputFile [1bestOutputFile]]\n (default to stdin/stdout)");
- System.exit(1);
- }
-
- try {
- // TODO: see documentation for extractOneBest
- // regarding using an n-best SegmentFileParser.
- IndexedReader<String> nbestReader =
- new IndexedReader<String>("line", "-".equals(inFile)
- ? new LineReader(System.in)
- : new LineReader(inFile));
-
- /*
- * TODO: This duplicates FileUtility.getWriteFileStream but with the addition of defaulting to
- * System.out; should fix that (without breaking other clients of that method). We ultimately
- * want something which autochecks for errors (like Writer); has a newLine method (like
- * BufferedWriter); can wrap System.out; can autoflush; and it'd be handy to have the
- * print/println methods of PrintStream/PrintWriter to boot. PrintWriter *almost* gives us all
- * this, but it swallows errors and gives no way to retrieve them >:(
- */
- BufferedWriter onebestWriter =
- new BufferedWriter(new OutputStreamWriter(("-".equals(outFile)
- ? System.out
- : new FileOutputStream(outFile, false)), "UTF-8"));
-
- extractOneBest(nbestReader, onebestWriter, field);
-
- } catch (IOException ioe) {
- // NOTE: if our onebest was System.out, then that
- // will already have been closed by the finally
- // block. Printing to a closed PrintStream generates
- // no exceptions. We should be printing to System.err
- // anyways, but this something subtle to be aware of.
- System.err.println("There was an error: " + ioe.getMessage());
- }
- }
-
-
- /**
- * Prints the one-best translation for each segment ID from the reader as a line on the writer,
- * and closes both before exiting. The translations for a segment are printed in the order of the
- * first occurance of the segment ID. Any information about the segment other than the translation
- * (including segment ID) is not printed to the writer.
- *
- * <h4>Developer Notes</h4> This implementation assumes:
- * <ol>
- * <li>all translations for a segment are contiguous</li>
- * <li>the 1-best translation is the first one encountered.</li>
- * </ol>
- * We will need to alter the implementation if these assumptions no longer hold for the output of
- * JoshuaDecoder (or any sensible n-best format passed to this method).
- * <p>
- * We should switch to using an n-best {@link joshua.decoder.segment_file.SegmentFileParser} to
- * ensure future compatibility with being able to configure the output format of the decoder. The
- * MERT code needs such a SegmentFileParser anyways, so that will reduce the code duplication
- * between these two classes.
- */
- protected static void extractOneBest(IndexedReader<String> nbestReader,
- BufferedWriter onebestWriter, int field) throws IOException {
-
- try {
- String prevID = null;
- for (String line : nbestReader) {
-
- // pass empty lines through
- if (Regex.commentOrEmptyLine.matches(line)) {
- onebestWriter.newLine();
- continue;
- }
-
- String[] columns = Regex.threeBarsWithSpace.split(line);
-
- // We allow non-integer segment IDs because the
- // Segment interface does, and we have no reason
- // to add new restrictions.
- String newID = columns[0].trim();
-
- // We want to give the same error message
- // regardless of whether there's a leading space
- // or not. And, we don't want to accidentally
- // accept lines with lots and lots of columns.
- if ("".equals(newID) || newID.startsWith("|||")) {
- throw nbestReader.wrapIOException(new IOException("Malformed line, missing segment ID:\n"
- + line));
- }
-
- // Make sure there's a translation there too
- // TODO: good error message for when the second
- // "|||" doesn't have a following field, m/\|{3}\s*$/
- if (3 > columns.length) {
- throw nbestReader.wrapIOException(new IOException(
- "Malformed line, should have at least two \" ||| \":\n" + line));
- }
-
-
- if (null == prevID || !prevID.equals(newID)) {
- onebestWriter.write(columns[field], 0, columns[field].length());
- onebestWriter.newLine();
- onebestWriter.flush();
-
- prevID = newID;
- }
- }
- } finally {
- try {
- nbestReader.close();
- } finally {
- onebestWriter.close();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/8cdbc4b8/src/joshua/util/FileUtility.java
----------------------------------------------------------------------
diff --git a/src/joshua/util/FileUtility.java b/src/joshua/util/FileUtility.java
deleted file mode 100644
index 0685655..0000000
--- a/src/joshua/util/FileUtility.java
+++ /dev/null
@@ -1,314 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package joshua.util;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.Closeable;
-import java.io.File;
-import java.io.FileDescriptor;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
-import java.nio.charset.Charset;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Scanner;
-
-/**
- * utility functions for file operations
- *
- * @author Zhifei Li, <zh...@gmail.com>
- * @author wren ng thornton <wr...@users.sourceforge.net>
- * @since 28 February 2009
- */
-public class FileUtility {
- public static String DEFAULT_ENCODING = "UTF-8";
-
- /*
- * Note: charset name is case-agnostic "UTF-8" is the canonical name "UTF8", "unicode-1-1-utf-8"
- * are aliases Java doesn't distinguish utf8 vs UTF-8 like Perl does
- */
- private static final Charset FILE_ENCODING = Charset.forName(DEFAULT_ENCODING);
-
- /** Warning, will truncate/overwrite existing files */
- public static BufferedWriter getWriteFileStream(String filename) throws IOException {
- return new BufferedWriter(new OutputStreamWriter(
- // TODO: add GZIP
- filename.equals("-") ? new FileOutputStream(FileDescriptor.out) : new FileOutputStream(
- filename, false), FILE_ENCODING));
- }
-
- /**
- * Recursively delete the specified file or directory.
- *
- * @param f File or directory to delete
- * @return <code>true</code> if the specified file or directory was deleted, <code>false</code>
- * otherwise
- */
- public static boolean deleteRecursively(File f) {
- if (null != f) {
- if (f.isDirectory())
- for (File child : f.listFiles())
- deleteRecursively(child);
- return f.delete();
- } else {
- return false;
- }
- }
-
- /**
- * Writes data from the integer array to disk as raw bytes, overwriting the old file if present.
- *
- * @param data The integer array to write to disk.
- * @param filename The filename where the data should be written.
- * @throws IOException
- * @return the FileOutputStream on which the bytes were written
- */
- public static FileOutputStream writeBytes(int[] data, String filename) throws IOException {
- FileOutputStream out = new FileOutputStream(filename, false);
- writeBytes(data, out);
- return out;
- }
-
- /**
- * Writes data from the integer array to disk as raw bytes.
- *
- * @param data The integer array to write to disk.
- * @param out The output stream where the data should be written.
- * @throws IOException
- */
- public static void writeBytes(int[] data, OutputStream out) throws IOException {
-
- byte[] b = new byte[4];
-
- for (int word : data) {
- b[0] = (byte) ((word >>> 24) & 0xFF);
- b[1] = (byte) ((word >>> 16) & 0xFF);
- b[2] = (byte) ((word >>> 8) & 0xFF);
- b[3] = (byte) ((word >>> 0) & 0xFF);
-
- out.write(b);
- }
- }
-
- public static void copyFile(String srFile, String dtFile) throws IOException {
- try {
- File f1 = new File(srFile);
- File f2 = new File(dtFile);
- copyFile(f1, f2);
- } catch (FileNotFoundException ex) {
- System.out.println(ex.getMessage() + " in the specified directory.");
- System.exit(0);
- } catch (IOException e) {
- System.out.println(e.getMessage());
- }
- }
-
- public static void copyFile(File srFile, File dtFile) throws IOException {
- try {
-
- InputStream in = new FileInputStream(srFile);
-
- // For Append the file.
- // OutputStream out = new FileOutputStream(f2,true);
-
- // For Overwrite the file.
- OutputStream out = new FileOutputStream(dtFile);
-
- byte[] buf = new byte[1024];
- int len;
- while ((len = in.read(buf)) > 0) {
- out.write(buf, 0, len);
- }
- in.close();
- out.close();
- System.out.println("File copied.");
- } catch (FileNotFoundException ex) {
- System.out.println(ex.getMessage() + " in the specified directory.");
- System.exit(0);
- } catch (IOException e) {
- System.out.println(e.getMessage());
- }
- }
-
- static public boolean deleteFile(String fileName) {
-
- File f = new File(fileName);
-
- // Make sure the file or directory exists and isn't write protected
- if (!f.exists())
- System.out.println("Delete: no such file or directory: " + fileName);
-
- if (!f.canWrite())
- System.out.println("Delete: write protected: " + fileName);
-
- // If it is a directory, make sure it is empty
- if (f.isDirectory()) {
- String[] files = f.list();
- if (files.length > 0)
- System.out.println("Delete: directory not empty: " + fileName);
- }
-
- // Attempt to delete it
- boolean success = f.delete();
-
- if (!success)
- System.out.println("Delete: deletion failed");
-
- return success;
-
- }
-
- /**
- * Returns the base directory of the file. For example, dirname('/usr/local/bin/emacs') ->
- * '/usr/local/bin'
- */
- static public String dirname(String fileName) {
- if (fileName.indexOf(File.separator) != -1)
- return fileName.substring(0, fileName.lastIndexOf(File.separator));
-
- return ".";
- }
-
- public static void createFolderIfNotExisting(String folderName) {
- File f = new File(folderName);
- if (!f.isDirectory()) {
- System.out.println(" createFolderIfNotExisting -- Making directory: " + folderName);
- f.mkdirs();
- } else {
- System.out.println(" createFolderIfNotExisting -- Directory: " + folderName
- + " already existed");
- }
- }
-
- public static void closeCloseableIfNotNull(Closeable fileWriter) {
- if (fileWriter != null) {
- try {
- fileWriter.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }
-
- /**
- * Returns the directory were the program has been started,
- * the base directory you will implicitly get when specifying no
- * full path when e.g. opening a file
- * @return
- */
- public static String getWorkingDirectory() {
- return System.getProperty("user.dir");
- }
-
- /**
- * Method to handle standard IO xceptions. catch (Exception e) {Utility.handleIO_exception(e);}
- */
- public static void handleExceptions(Exception e) {
- e.printStackTrace();
- System.exit(-1);
- }
-
- /**
- * Convenience method to get a full file as a String
- * @param file
- * @return The file as a String. Lines are separated by newline character.
- */
- public static String getFileAsString(File file) {
- String result = "";
- List<String> lines = getLines(file, true);
- for (int i = 0; i < lines.size() - 1; i++) {
- result += lines.get(i) + "\n";
- }
- if (!lines.isEmpty()) {
- result += lines.get(lines.size() - 1);
- }
- return result;
- }
-
- /**
- * This method returns a List of String. Each element of the list corresponds to a line from the
- * input file. The boolean keepDuplicates in the input determines if duplicate lines are allowed
- * in the output LinkedList or not.
- */
- static public List<String> getLines(File file, boolean keepDuplicates) {
- LinkedList<String> list = new LinkedList<String>();
- String line = "";
- try {
- BufferedReader InputReader = new BufferedReader(new FileReader(file));
- for (;;) { // this loop writes writes in a Sting each sentence of
- // the file and process it
- int current = InputReader.read();
- if (current == -1 || current == '\n') {
- if (keepDuplicates || !list.contains(line))
- list.add(line);
- line = "";
- if (current == -1)
- break; // EOF
- } else
- line += (char) current;
- }
- InputReader.close();
- } catch (Exception e) {
- handleExceptions(e);
- }
- return list;
- }
-
- /**
- * Returns a Scanner of the inputFile using a specific encoding
- *
- * @param inputFile
- * @return : Scanner
- */
- public static Scanner getScanner(File inputFile, String encoding) {
- Scanner scan = null;
- try {
- scan = new Scanner(inputFile, encoding);
- } catch (IOException e) {
- FileUtility.handleExceptions(e);
- }
- return scan;
- }
-
- /**
- * Returns a Scanner of the inputFile using default encoding
- *
- * @param inputFile
- * @return : Scanner
- */
- public static Scanner getScanner(File inputFile) {
- return getScanner(inputFile, DEFAULT_ENCODING);
- }
-
- static public String getFirstLineInFile(File inputFile) {
- Scanner scan = FileUtility.getScanner(inputFile);
- if (!scan.hasNextLine())
- return null;
- String line = scan.nextLine();
- scan.close();
- return line;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/8cdbc4b8/src/joshua/util/FormatUtils.java
----------------------------------------------------------------------
diff --git a/src/joshua/util/FormatUtils.java b/src/joshua/util/FormatUtils.java
deleted file mode 100644
index 67b2bf3..0000000
--- a/src/joshua/util/FormatUtils.java
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package joshua.util;
-
-import java.io.PrintStream;
-import java.io.UnsupportedEncodingException;
-import java.util.regex.Pattern;
-
-import joshua.corpus.Vocabulary;
-
-/**
- * Utility class for format issues.
- *
- * @author Juri Ganitkevitch
- * @author Lane Schwartz
- */
-public class FormatUtils {
-
- private static final String INDEX_SEPARATOR = ",";
-
- /**
- * Determines whether the string is a nonterminal by checking that the first character is [
- * and the last character is ].
- *
- * @param token
- * @return true if it's a nonterminal symbol, false otherwise
- */
- public static boolean isNonterminal(String token) {
- return (token.length() >=3 && token.charAt(0) == '[') && (token.charAt(token.length() - 1) == ']');
- }
-
- /**
- * Nonterminals are stored in the vocabulary in square brackets. This removes them when you
- * just want the raw nonterminal word.
- * Supports indexed and non-indexed nonTerminals:
- * [GOAL] -> GOAL
- * [X,1] -> [X]
- *
- * @param nt the nonterminal, e.g., "[GOAL]"
- * @return the cleaned nonterminal, e.g., "GOAL"
- */
- public static String cleanNonTerminal(String nt) {
- if (isNonterminal(nt)) {
- if (isIndexedNonTerminal(nt)) {
- // strip ",.*]"
- return nt.substring(1, nt.indexOf(INDEX_SEPARATOR));
- }
- // strip "]"
- return nt.substring(1, nt.length() - 1);
- }
- return nt;
- }
-
- private static boolean isIndexedNonTerminal(String nt) {
- return nt.contains(INDEX_SEPARATOR);
- }
-
- /**
- * Removes the index from a nonTerminal: [X,1] -> [X].
- */
- public static String stripNonTerminalIndex(String nt) {
- return markup(cleanNonTerminal(nt));
- }
-
- public static int getNonterminalIndex(String nt) {
- return Integer.parseInt(nt.substring(nt.indexOf(INDEX_SEPARATOR) + 1, nt.length() - 1));
- }
-
- /**
- * Ensures that a string looks like what the system considers a nonterminal to be.
- *
- * @param nt the nonterminal string
- * @return the nonterminal string surrounded in square brackets (if not already)
- */
- public static String markup(String nt) {
- if (isNonterminal(nt))
- return nt;
- else
- return "[" + nt + "]";
- }
-
- public static String markup(String nt, int index) {
- if (isNonterminal(nt)) {
- return markup(cleanNonTerminal(nt), index);
- }
- return "[" + nt + INDEX_SEPARATOR + index + "]";
- }
-
- public static String escapeSpecialSymbols(String s) {
- return s.replaceAll("\\[", "-lsb-")
- .replaceAll("\\]", "-rsb-")
- .replaceAll("\\|", "-pipe-");
- }
-
- public static String unescapeSpecialSymbols(String s) {
- return s.replaceAll("-lsb-", "[")
- .replaceAll("-rsb-", "]")
- .replaceAll("-pipe-", "|");
- }
-
- /**
- * wrap sentence with sentence start/stop markers
- * as defined by Vocabulary; separated by a single whitespace.
- */
- public static String addSentenceMarkers(String s) {
- return Vocabulary.START_SYM + " " + s + " " + Vocabulary.STOP_SYM;
- }
-
- /**
- * strip sentence markers (and whitespaces) from string
- */
- public static String removeSentenceMarkers(String s) {
- return s.replaceAll("<s> ", "").replace(" </s>", "");
- }
-
- /**
- * Returns true if the String parameter represents a valid number.
- * <p>
- * The body of this method is taken from the Javadoc documentation for the Java Double class.
- *
- * @param string
- * @see java.lang.Double
- * @return <code>true</code> if the string represents a valid number, <code>false</code> otherwise
- */
- public static boolean isNumber(String string) {
- final String Digits = "(\\p{Digit}+)";
- final String HexDigits = "(\\p{XDigit}+)";
- // an exponent is 'e' or 'E' followed by an optionally
- // signed decimal integer.
- final String Exp = "[eE][+-]?" + Digits;
- final String fpRegex = ("[\\x00-\\x20]*" + // Optional leading "whitespace"
- "[+-]?(" + // Optional sign character
- "NaN|" + // "NaN" string
- "Infinity|" + // "Infinity" string
-
- // A decimal floating-point string representing a finite positive
- // number without a leading sign has at most five basic pieces:
- // Digits . Digits ExponentPart FloatTypeSuffix
- //
- // Since this method allows integer-only strings as input
- // in addition to strings of floating-point literals, the
- // two sub-patterns below are simplifications of the grammar
- // productions from the Java Language Specification, 2nd
- // edition, section 3.10.2.
-
- // Digits ._opt Digits_opt ExponentPart_opt FloatTypeSuffix_opt
- "(((" + Digits + "(\\.)?(" + Digits + "?)(" + Exp + ")?)|" +
-
- // . Digits ExponentPart_opt FloatTypeSuffix_opt
- "(\\.(" + Digits + ")(" + Exp + ")?)|" +
-
- // Hexadecimal strings
- "((" +
- // 0[xX] HexDigits ._opt BinaryExponent FloatTypeSuffix_opt
- "(0[xX]" + HexDigits + "(\\.)?)|" +
-
- // 0[xX] HexDigits_opt . HexDigits BinaryExponent FloatTypeSuffix_opt
- "(0[xX]" + HexDigits + "?(\\.)" + HexDigits + ")" +
-
- ")[pP][+-]?" + Digits + "))" + "[fFdD]?))" + "[\\x00-\\x20]*");// Optional
- // trailing
- // "whitespace"
-
- return Pattern.matches(fpRegex, string);
- }
-
- /**
- * Set System.out and System.err to use the UTF8 character encoding.
- *
- * @return <code>true</code> if both System.out and System.err were successfully set to use UTF8,
- * <code>false</code> otherwise.
- */
- public static boolean useUTF8() {
-
- try {
- System.setOut(new PrintStream(System.out, true, "UTF8"));
- System.setErr(new PrintStream(System.err, true, "UTF8"));
- return true;
- } catch (UnsupportedEncodingException e1) {
- System.err
- .println("UTF8 is not a valid encoding; using system default encoding for System.out and System.err.");
- return false;
- } catch (SecurityException e2) {
- System.err
- .println("Security manager is configured to disallow changes to System.out or System.err; using system default encoding.");
- return false;
- }
- }
-
- /**
- * Determines if a string contains ALL CAPS
- *
- * @param token
- * @return true if the string is all in uppercase, false otherwise
- */
- public static boolean ISALLUPPERCASE(String token) {
- for (int i = 0; i < token.length(); i++)
- if (! Character.isUpperCase(token.charAt(i)))
- return false;
- return true;
- }
-
- public static String capitalize(String word) {
- if (word == null || word.length() == 0)
- return word;
- return word.substring(0, 1).toUpperCase() + word.substring(1);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/8cdbc4b8/src/joshua/util/IntegerPair.java
----------------------------------------------------------------------
diff --git a/src/joshua/util/IntegerPair.java b/src/joshua/util/IntegerPair.java
deleted file mode 100644
index 08cefe1..0000000
--- a/src/joshua/util/IntegerPair.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package joshua.util;
-
-/**
- * Memory-efficient implementation of an integer tuple.
- *
- * @author Lane Schwartz
- */
-public final class IntegerPair {
-
- public final int first;
- public final int second;
-
- public IntegerPair(final int first, final int second) {
- this.first = first;
- this.second = second;
- }
-
-}