You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/04/19 21:33:49 UTC
[01/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Repository: incubator-joshua
Updated Branches:
refs/heads/master 76bb8fdcf -> 6da3961be
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/file_piece.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/file_piece.hh b/ext/kenlm/util/file_piece.hh
deleted file mode 100644
index d3d8305..0000000
--- a/ext/kenlm/util/file_piece.hh
+++ /dev/null
@@ -1,175 +0,0 @@
-#ifndef UTIL_FILE_PIECE_H
-#define UTIL_FILE_PIECE_H
-
-#include "util/ersatz_progress.hh"
-#include "util/exception.hh"
-#include "util/file.hh"
-#include "util/mmap.hh"
-#include "util/read_compressed.hh"
-#include "util/string_piece.hh"
-
-#include <cstddef>
-#include <iosfwd>
-#include <string>
-#include <cassert>
-#include <stdint.h>
-
-namespace util {
-
-class ParseNumberException : public Exception {
- public:
- explicit ParseNumberException(StringPiece value) throw();
- ~ParseNumberException() throw() {}
-};
-
-extern const bool kSpaces[256];
-
-// Memory backing the returned StringPiece may vanish on the next call.
-class FilePiece {
- public:
- // 1 MB default.
- explicit FilePiece(const char *file, std::ostream *show_progress = NULL, std::size_t min_buffer = 1048576);
- // Takes ownership of fd. name is used for messages.
- explicit FilePiece(int fd, const char *name = NULL, std::ostream *show_progress = NULL, std::size_t min_buffer = 1048576);
-
- /* Read from an istream. Don't use this if you can avoid it. Raw fd IO is
- * much faster. But sometimes you just have an istream like Boost's HTTP
- * server and want to parse it the same way.
- * name is just used for messages and FileName().
- */
- explicit FilePiece(std::istream &stream, const char *name = NULL, std::size_t min_buffer = 1048576);
-
- ~FilePiece();
-
- char get() {
- if (position_ == position_end_) {
- Shift();
- if (at_end_) throw EndOfFileException();
- }
- return *(position_++);
- }
-
- // Leaves the delimiter, if any, to be returned by get(). Delimiters defined by isspace().
- StringPiece ReadDelimited(const bool *delim = kSpaces) {
- SkipSpaces(delim);
- return Consume(FindDelimiterOrEOF(delim));
- }
-
- /// Read word until the line or file ends.
- bool ReadWordSameLine(StringPiece &to, const bool *delim = kSpaces) {
- assert(delim[static_cast<unsigned char>('\n')]);
- // Skip non-enter spaces.
- for (; ; ++position_) {
- if (position_ == position_end_) {
- try {
- Shift();
- } catch (const util::EndOfFileException &e) { return false; }
- // And break out at end of file.
- if (position_ == position_end_) return false;
- }
- if (!delim[static_cast<unsigned char>(*position_)]) break;
- if (*position_ == '\n') return false;
- }
- // We can't be at the end of file because there's at least one character open.
- to = Consume(FindDelimiterOrEOF(delim));
- return true;
- }
-
- /** Read a line of text from the file.
- *
- * Unlike ReadDelimited, this includes leading spaces and consumes the
- * delimiter. It is similar to getline in that way.
- *
- * If strip_cr is true, any trailing carriate return (as would be found on
- * a file written on Windows) will be left out of the returned line.
- *
- * Throws EndOfFileException if the end of the file is encountered. If the
- * file does not end in a newline, this could mean that the last line is
- * never read.
- */
- StringPiece ReadLine(char delim = '\n', bool strip_cr = true);
-
- /** Read a line of text from the file, or return false on EOF.
- *
- * This is like ReadLine, except it returns false where ReadLine throws
- * EndOfFileException. Like ReadLine it may not read the last line in the
- * file if the file does not end in a newline.
- *
- * If strip_cr is true, any trailing carriate return (as would be found on
- * a file written on Windows) will be left out of the returned line.
- */
- bool ReadLineOrEOF(StringPiece &to, char delim = '\n', bool strip_cr = true);
-
- float ReadFloat();
- double ReadDouble();
- long int ReadLong();
- unsigned long int ReadULong();
-
- // Skip spaces defined by isspace.
- void SkipSpaces(const bool *delim = kSpaces) {
- assert(position_ <= position_end_);
- for (; ; ++position_) {
- if (position_ == position_end_) {
- Shift();
- // And break out at end of file.
- if (position_ == position_end_) return;
- }
- assert(position_ < position_end_);
- if (!delim[static_cast<unsigned char>(*position_)]) return;
- }
- }
-
- uint64_t Offset() const {
- return position_ - data_.begin() + mapped_offset_;
- }
-
- const std::string &FileName() const { return file_name_; }
-
- private:
- void InitializeNoRead(const char *name, std::size_t min_buffer);
- // Calls InitializeNoRead, so don't call both.
- void Initialize(const char *name, std::ostream *show_progress, std::size_t min_buffer);
-
- template <class T> T ReadNumber();
-
- StringPiece Consume(const char *to) {
- assert(to >= position_);
- StringPiece ret(position_, to - position_);
- position_ = to;
- return ret;
- }
-
- const char *FindDelimiterOrEOF(const bool *delim = kSpaces);
-
- void Shift();
- // Backends to Shift().
- void MMapShift(uint64_t desired_begin);
-
- void TransitionToRead();
- void ReadShift();
-
- const char *position_, *last_space_, *position_end_;
-
- scoped_fd file_;
- const uint64_t total_size_;
- const uint64_t page_;
-
- std::size_t default_map_size_;
- uint64_t mapped_offset_;
-
- // Order matters: file_ should always be destroyed after this.
- scoped_memory data_;
-
- bool at_end_;
- bool fallback_to_read_;
-
- ErsatzProgress progress_;
-
- std::string file_name_;
-
- ReadCompressed fell_back_;
-};
-
-} // namespace util
-
-#endif // UTIL_FILE_PIECE_H
[11/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/pipeline.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/pipeline.cc b/ext/kenlm/lm/interpolate/pipeline.cc
deleted file mode 100644
index 47b8288..0000000
--- a/ext/kenlm/lm/interpolate/pipeline.cc
+++ /dev/null
@@ -1,159 +0,0 @@
-#include "lm/interpolate/pipeline.hh"
-
-#include "lm/common/compare.hh"
-#include "lm/common/print.hh"
-#include "lm/common/renumber.hh"
-#include "lm/vocab.hh"
-#include "lm/interpolate/backoff_reunification.hh"
-#include "lm/interpolate/interpolate_info.hh"
-#include "lm/interpolate/merge_probabilities.hh"
-#include "lm/interpolate/merge_vocab.hh"
-#include "lm/interpolate/normalize.hh"
-#include "lm/interpolate/universal_vocab.hh"
-#include "util/stream/chain.hh"
-#include "util/stream/count_records.hh"
-#include "util/stream/io.hh"
-#include "util/stream/multi_stream.hh"
-#include "util/stream/sort.hh"
-#include "util/fixed_array.hh"
-
-namespace lm { namespace interpolate { namespace {
-
-/* Put the original input files on chains and renumber them */
-void SetupInputs(std::size_t buffer_size, const UniversalVocab &vocab, util::FixedArray<ModelBuffer> &models, bool exclude_highest, util::FixedArray<util::stream::Chains> &chains, util::FixedArray<util::stream::ChainPositions> &positions) {
- chains.clear();
- positions.clear();
- // TODO: much better memory sizing heuristics e.g. not making the chain larger than it will use.
- util::stream::ChainConfig config(0, 2, buffer_size);
- for (std::size_t i = 0; i < models.size(); ++i) {
- chains.push_back(models[i].Order() - exclude_highest);
- for (std::size_t j = 0; j < models[i].Order() - exclude_highest; ++j) {
- config.entry_size = sizeof(WordIndex) * (j + 1) + sizeof(float) * 2; // TODO do not include wasteful backoff for highest.
- chains.back().push_back(config);
- }
- models[i].Source(chains.back());
- for (std::size_t j = 0; j < models[i].Order() - exclude_highest; ++j) {
- chains[i][j] >> Renumber(vocab.Mapping(i), j + 1);
- }
- }
- for (std::size_t i = 0; i < chains.size(); ++i) {
- positions.push_back(chains[i]);
- }
-}
-
-template <class SortOrder> void ApplySort(const util::stream::SortConfig &config, util::stream::Chains &chains) {
- util::stream::Sorts<SortOrder> sorts(chains.size());
- for (std::size_t i = 0; i < chains.size(); ++i) {
- sorts.push_back(chains[i], config, SortOrder(i + 1));
- }
- chains.Wait(true);
- // TODO memory management
- for (std::size_t i = 0; i < sorts.size(); ++i) {
- sorts[i].Merge(sorts[i].DefaultLazy());
- }
- for (std::size_t i = 0; i < sorts.size(); ++i) {
- sorts[i].Output(chains[i], sorts[i].DefaultLazy());
- }
-};
-
-} // namespace
-
-void Pipeline(util::FixedArray<ModelBuffer> &models, const Config &config, int write_file) {
- // Setup InterpolateInfo and UniversalVocab.
- InterpolateInfo info;
- info.lambdas = config.lambdas;
- std::vector<WordIndex> vocab_sizes;
-
- util::scoped_fd vocab_null(util::MakeTemp(config.sort.temp_prefix));
- std::size_t max_order = 0;
- util::FixedArray<util::scoped_fd> vocab_files(models.size());
- for (ModelBuffer *i = models.begin(); i != models.end(); ++i) {
- info.orders.push_back(i->Order());
- vocab_sizes.push_back(i->Counts()[0]);
- vocab_files.push_back(util::DupOrThrow(i->VocabFile()));
- max_order = std::max(max_order, i->Order());
- }
- UniversalVocab vocab(vocab_sizes);
- {
- ngram::ImmediateWriteWordsWrapper writer(NULL, vocab_null.get(), 0);
- MergeVocab(vocab_files, vocab, writer);
- }
- vocab_files.clear();
-
- std::cerr << "Merging probabilities." << std::endl;
- // Pass 1: merge probabilities
- util::FixedArray<util::stream::Chains> input_chains(models.size());
- util::FixedArray<util::stream::ChainPositions> models_by_order(models.size());
- SetupInputs(config.BufferSize(), vocab, models, false, input_chains, models_by_order);
-
- util::stream::Chains merged_probs(max_order);
- for (std::size_t i = 0; i < max_order; ++i) {
- merged_probs.push_back(util::stream::ChainConfig(PartialProbGamma::TotalSize(info, i + 1), 2, config.BufferSize())); // TODO: not buffer_size
- }
- MergeProbabilities(info, models_by_order, merged_probs);
- std::vector<uint64_t> counts(max_order);
- for (std::size_t i = 0; i < max_order; ++i) {
- merged_probs[i] >> util::stream::CountRecords(&counts[i]);
- }
-
- // Pass 2: normalize.
- ApplySort<ContextOrder>(config.sort, merged_probs);
- std::cerr << "Normalizing" << std::endl;
- SetupInputs(config.BufferSize(), vocab, models, true, input_chains, models_by_order);
- util::stream::Chains probabilities(max_order), backoffs(max_order - 1);
- std::size_t block_count = 2;
- for (std::size_t i = 0; i < max_order; ++i) {
- // Careful accounting to ensure RewindableStream can fit the entire vocabulary.
- block_count = std::max<std::size_t>(block_count, 2);
- // This much needs to fit in RewindableStream.
- std::size_t fit = NGram<float>::TotalSize(i + 1) * counts[0];
- // fit / (block_count - 1) rounded up
- std::size_t min_block = (fit + block_count - 2) / (block_count - 1);
- std::size_t specify = std::max(config.BufferSize(), min_block * block_count);
- probabilities.push_back(util::stream::ChainConfig(NGram<float>::TotalSize(i + 1), block_count, specify));
- }
- for (std::size_t i = 0; i < max_order - 1; ++i) {
- backoffs.push_back(util::stream::ChainConfig(sizeof(float), 2, config.BufferSize()));
- }
- Normalize(info, models_by_order, merged_probs, probabilities, backoffs);
-
- util::FixedArray<util::stream::FileBuffer> backoff_buffers(backoffs.size());
- for (std::size_t i = 0; i < max_order - 1; ++i) {
- backoff_buffers.push_back(util::MakeTemp(config.sort.temp_prefix));
- backoffs[i] >> backoff_buffers.back().Sink();
- }
-
- // Pass 3: backoffs in the right place.
- ApplySort<SuffixOrder>(config.sort, probabilities);
- // TODO destroy universal vocab to save RAM.
- // TODO these should be freed before merge sort happens in the above function.
- backoffs.Wait(true);
- merged_probs.Wait(true);
- std::cerr << "Reunifying backoffs" << std::endl;
-
- util::stream::ChainPositions prob_pos(max_order - 1);
- util::stream::Chains combined(max_order - 1);
- for (std::size_t i = 0; i < max_order - 1; ++i) {
- backoffs[i] >> backoff_buffers[i].Source(true);
- prob_pos.push_back(probabilities[i].Add());
- combined.push_back(util::stream::ChainConfig(NGram<ProbBackoff>::TotalSize(i + 1), 2, config.BufferSize()));
- }
- util::stream::ChainPositions backoff_pos(backoffs);
-
- ReunifyBackoff(prob_pos, backoff_pos, combined);
-
- util::stream::ChainPositions output_pos(max_order);
- for (std::size_t i = 0; i < max_order - 1; ++i) {
- output_pos.push_back(combined[i].Add());
- }
- output_pos.push_back(probabilities.back().Add());
-
- probabilities >> util::stream::kRecycle;
- backoffs >> util::stream::kRecycle;
- combined >> util::stream::kRecycle;
-
- // TODO genericize to ModelBuffer etc.
- PrintARPA(vocab_null.get(), write_file, counts).Run(output_pos);
-}
-
-}} // namespaces
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/pipeline.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/pipeline.hh b/ext/kenlm/lm/interpolate/pipeline.hh
deleted file mode 100644
index b200248..0000000
--- a/ext/kenlm/lm/interpolate/pipeline.hh
+++ /dev/null
@@ -1,22 +0,0 @@
-#ifndef LM_INTERPOLATE_PIPELINE_H
-#define LM_INTERPOLATE_PIPELINE_H
-
-#include "lm/common/model_buffer.hh"
-#include "util/fixed_array.hh"
-#include "util/stream/config.hh"
-
-#include <cstddef>
-#include <string>
-
-namespace lm { namespace interpolate {
-
-struct Config {
- std::vector<float> lambdas;
- util::stream::SortConfig sort;
- std::size_t BufferSize() const { return sort.buffer_size; }
-};
-
-void Pipeline(util::FixedArray<ModelBuffer> &models, const Config &config, int write_file);
-
-}} // namespaces
-#endif // LM_INTERPOLATE_PIPELINE_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/split_worker.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/split_worker.cc b/ext/kenlm/lm/interpolate/split_worker.cc
deleted file mode 100644
index e777bf0..0000000
--- a/ext/kenlm/lm/interpolate/split_worker.cc
+++ /dev/null
@@ -1,40 +0,0 @@
-#include "lm/interpolate/split_worker.hh"
-#include "lm/common/ngram.hh"
-
-namespace lm {
-namespace interpolate {
-
-SplitWorker::SplitWorker(std::size_t order, util::stream::Chain &backoff_chain,
- util::stream::Chain &sort_chain)
- : order_(order) {
- backoff_chain >> backoff_input_;
- sort_chain >> sort_input_;
-}
-
-void SplitWorker::Run(const util::stream::ChainPosition &position) {
- // input: ngram record (id, prob, and backoff)
- // output: a float to the backoff_input stream
- // an ngram id and a float to the sort_input stream
- for (util::stream::Stream stream(position); stream; ++stream) {
- NGram<ProbBackoff> ngram(stream.Get(), order_);
-
- // write id and prob to the sort stream
- float prob = ngram.Value().prob;
- lm::WordIndex *out = reinterpret_cast<lm::WordIndex *>(sort_input_.Get());
- for (const lm::WordIndex *it = ngram.begin(); it != ngram.end(); ++it) {
- *out++ = *it;
- }
- *reinterpret_cast<float *>(out) = prob;
- ++sort_input_;
-
- // write backoff to the backoff output stream
- float boff = ngram.Value().backoff;
- *reinterpret_cast<float *>(backoff_input_.Get()) = boff;
- ++backoff_input_;
- }
- sort_input_.Poison();
- backoff_input_.Poison();
-}
-
-}
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/split_worker.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/split_worker.hh b/ext/kenlm/lm/interpolate/split_worker.hh
deleted file mode 100644
index 15fae68..0000000
--- a/ext/kenlm/lm/interpolate/split_worker.hh
+++ /dev/null
@@ -1,44 +0,0 @@
-#ifndef KENLM_INTERPOLATE_SPLIT_WORKER_H_
-#define KENLM_INTERPOLATE_SPLIT_WORKER_H_
-
-#include "util/stream/chain.hh"
-#include "util/stream/stream.hh"
-
-namespace lm {
-namespace interpolate {
-
-class SplitWorker {
- public:
- /**
- * Constructs a split worker for a particular order. It writes the
- * split-off backoff values to the backoff chain and the ngram id and
- * probability to the sort chain for each ngram in the input.
- */
- SplitWorker(std::size_t order, util::stream::Chain &backoff_chain,
- util::stream::Chain &sort_chain);
-
- /**
- * The callback invoked to handle the input from the ngram intermediate
- * files.
- */
- void Run(const util::stream::ChainPosition& position);
-
- private:
- /**
- * The ngram order we are reading/writing for.
- */
- std::size_t order_;
-
- /**
- * The stream to write to for the backoff values.
- */
- util::stream::Stream backoff_input_;
-
- /**
- * The stream to write to for the ngram id + probability values.
- */
- util::stream::Stream sort_input_;
-};
-}
-}
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/streaming_example_main.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/streaming_example_main.cc b/ext/kenlm/lm/interpolate/streaming_example_main.cc
deleted file mode 100644
index 1f543cb..0000000
--- a/ext/kenlm/lm/interpolate/streaming_example_main.cc
+++ /dev/null
@@ -1,195 +0,0 @@
-#include "lm/common/compare.hh"
-#include "lm/common/model_buffer.hh"
-#include "lm/common/ngram.hh"
-#include "util/stream/chain.hh"
-#include "util/stream/multi_stream.hh"
-#include "util/stream/sort.hh"
-#include "lm/interpolate/split_worker.hh"
-
-#include <boost/program_options.hpp>
-#include <boost/version.hpp>
-
-#if defined(_WIN32) || defined(_WIN64)
-
-// Windows doesn't define <unistd.h>
-//
-// So we define what we need here instead:
-//
-#define STDIN_FILENO = 0
-#define STDOUT_FILENO = 1
-#else // Huzzah for POSIX!
-#include <unistd.h>
-#endif
-
-/*
- * This is a simple example program that takes in intermediate
- * suffix-sorted ngram files and outputs two sets of files: one for backoff
- * probability values (raw numbers, in suffix order) and one for
- * probability values (ngram id and probability, in *context* order)
- */
-int main(int argc, char *argv[]) {
- using namespace lm::interpolate;
-
- const std::size_t ONE_GB = 1 << 30;
- const std::size_t SIXTY_FOUR_MB = 1 << 26;
- const std::size_t NUMBER_OF_BLOCKS = 2;
-
- std::string FILE_NAME = "ngrams";
- std::string CONTEXT_SORTED_FILENAME = "csorted-ngrams";
- std::string BACKOFF_FILENAME = "backoffs";
- std::string TMP_DIR = "/tmp/";
-
- try {
- namespace po = boost::program_options;
- po::options_description options("canhazinterp Pass-3 options");
-
- options.add_options()
- ("help,h", po::bool_switch(), "Show this help message")
- ("ngrams,n", po::value<std::string>(&FILE_NAME), "ngrams file")
- ("csortngrams,c", po::value<std::string>(&CONTEXT_SORTED_FILENAME), "context sorted ngrams file")
- ("backoffs,b", po::value<std::string>(&BACKOFF_FILENAME), "backoffs file")
- ("tmpdir,t", po::value<std::string>(&TMP_DIR), "tmp dir");
- po::variables_map vm;
- po::store(po::parse_command_line(argc, argv, options), vm);
-
- // Display help
- if(vm["help"].as<bool>()) {
- std::cerr << "Usage: " << options << std::endl;
- return 1;
- }
- }
- catch(const std::exception &e) {
-
- std::cerr << e.what() << std::endl;
- return 1;
-
- }
-
- // The basic strategy here is to have three chains:
- // - The first reads the ngram order inputs using ModelBuffer. Those are
- // then stripped of their backoff values and fed into the third chain;
- // the backoff values *themselves* are written to the second chain.
- //
- // - The second chain takes the backoff values and writes them out to a
- // file (one for each order).
- //
- // - The third chain takes just the probability values and ngrams and
- // writes them out, sorted in context-order, to a file (one for each
- // order).
-
- // This will be used to read in the binary intermediate files. There is
- // one file per order (e.g. ngrams.1, ngrams.2, ...)
- lm::ModelBuffer buffer(FILE_NAME);
-
- // Create a separate chains for each ngram order for:
- // - Input from the intermediate files
- // - Output to the backoff file
- // - Output to the (context-sorted) probability file
- util::stream::Chains ngram_inputs(buffer.Order());
- util::stream::Chains backoff_chains(buffer.Order());
- util::stream::Chains prob_chains(buffer.Order());
- for (std::size_t i = 0; i < buffer.Order(); ++i) {
- ngram_inputs.push_back(util::stream::ChainConfig(
- lm::NGram<lm::ProbBackoff>::TotalSize(i + 1), NUMBER_OF_BLOCKS, ONE_GB));
-
- backoff_chains.push_back(
- util::stream::ChainConfig(sizeof(float), NUMBER_OF_BLOCKS, ONE_GB));
-
- prob_chains.push_back(util::stream::ChainConfig(
- sizeof(lm::WordIndex) * (i + 1) + sizeof(float), NUMBER_OF_BLOCKS,
- ONE_GB));
- }
-
- // This sets the input for each of the ngram order chains to the
- // appropriate file
- buffer.Source(ngram_inputs);
-
- util::FixedArray<util::scoped_ptr<SplitWorker> > workers(buffer.Order());
- for (std::size_t i = 0; i < buffer.Order(); ++i) {
- // Attach a SplitWorker to each of the ngram input chains, writing to the
- // corresponding order's backoff and probability chains
- workers.push_back(
- new SplitWorker(i + 1, backoff_chains[i], prob_chains[i]));
- ngram_inputs[i] >> boost::ref(*workers.back());
- }
-
- util::stream::SortConfig sort_cfg;
- sort_cfg.temp_prefix = TMP_DIR;
- sort_cfg.buffer_size = SIXTY_FOUR_MB;
- sort_cfg.total_memory = ONE_GB;
-
- // This will parallel merge sort the individual order files, putting
- // them in context-order instead of suffix-order.
- //
- // Two new threads will be running, each owned by the chains[i] object.
- // - The first executes BlockSorter.Run() to sort the n-gram entries
- // - The second executes WriteAndRecycle.Run() to write each sorted
- // block to disk as a temporary file
- util::stream::Sorts<lm::ContextOrder> sorts(buffer.Order());
- for (std::size_t i = 0; i < prob_chains.size(); ++i) {
- sorts.push_back(prob_chains[i], sort_cfg, lm::ContextOrder(i + 1));
- }
-
- // Set the sort output to be on the same chain
- for (std::size_t i = 0; i < prob_chains.size(); ++i) {
- // The following call to Chain::Wait()
- // joins the threads owned by chains[i].
- //
- // As such the following call won't return
- // until all threads owned by chains[i] have completed.
- //
- // The following call also resets chain[i]
- // so that it can be reused
- // (including free'ing the memory previously used by the chain)
- prob_chains[i].Wait();
-
- // In an ideal world (without memory restrictions)
- // we could merge all of the previously sorted blocks
- // by reading them all completely into memory
- // and then running merge sort over them.
- //
- // In the real world, we have memory restrictions;
- // depending on how many blocks we have,
- // and how much memory we can use to read from each block
- // (sort_config.buffer_size)
- // it may be the case that we have insufficient memory
- // to read sort_config.buffer_size of data from each block from disk.
- //
- // If this occurs, then it will be necessary to perform one or more rounds
- // of merge sort on disk;
- // doing so will reduce the number of blocks that we will eventually
- // need to read from
- // when performing the final round of merge sort in memory.
- //
- // So, the following call determines whether it is necessary
- // to perform one or more rounds of merge sort on disk;
- // if such on-disk merge sorting is required, such sorting is performed.
- //
- // Finally, the following method launches a thread that calls
- // OwningMergingReader.Run()
- // to perform the final round of merge sort in memory.
- //
- // Merge sort could have be invoked directly
- // so that merge sort memory doesn't coexist with Chain memory.
- sorts[i].Output(prob_chains[i]);
- }
-
- // Create another model buffer for our output on e.g. csorted-ngrams.1,
- // csorted-ngrams.2, ...
- lm::ModelBuffer output_buf(CONTEXT_SORTED_FILENAME, true, false);
- output_buf.Sink(prob_chains, buffer.Counts());
-
- // Create a third model buffer for our backoff output on e.g. backoff.1,
- // backoff.2, ...
- lm::ModelBuffer boff_buf(BACKOFF_FILENAME, true, false);
- boff_buf.Sink(backoff_chains, buffer.Counts());
-
- // Joins all threads that chains owns,
- // and does a for loop over each chain object in chains,
- // calling chain.Wait() on each such chain object
- ngram_inputs.Wait(true);
- backoff_chains.Wait(true);
- prob_chains.Wait(true);
-
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/toy_data/toy.linear_interpolation.lambda1_0.4.lambda2_0.6.lm
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/toy_data/toy.linear_interpolation.lambda1_0.4.lambda2_0.6.lm b/ext/kenlm/lm/interpolate/toy_data/toy.linear_interpolation.lambda1_0.4.lambda2_0.6.lm
deleted file mode 100644
index f4bac3d..0000000
--- a/ext/kenlm/lm/interpolate/toy_data/toy.linear_interpolation.lambda1_0.4.lambda2_0.6.lm
+++ /dev/null
@@ -1,23 +0,0 @@
-
-\data\
-ngram 1=5
-ngram 2=8
-
-\1-grams:
--0.5850267 </s>
--99 <s> -7.004176
--0.7447274 <unk>
--0.4685211 a -7.272811
--0.6575773 b -99
-
-\2-grams:
--0.2839967 <s> a
--0.3187588 <s> b
--0.79588 a </s>
--0.5058454 a a
--0.2773661 a b
--0.7447275 b </s>
--0.1135093 b a
--0.9586073 b b
-
-\end\
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/toy_data/toy.loglinear_interpolation.lambda1_0.4.lambda2_0.6.lm
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/toy_data/toy.loglinear_interpolation.lambda1_0.4.lambda2_0.6.lm b/ext/kenlm/lm/interpolate/toy_data/toy.loglinear_interpolation.lambda1_0.4.lambda2_0.6.lm
deleted file mode 100644
index 9874ba9..0000000
--- a/ext/kenlm/lm/interpolate/toy_data/toy.loglinear_interpolation.lambda1_0.4.lambda2_0.6.lm
+++ /dev/null
@@ -1,23 +0,0 @@
-
-\data\
-ngram 1=5
-ngram 2=8
-
-\1-grams:
--0.6190513 </s>
--99 <s> -6.885247
--99 <unk>
--0.3394633 a 0
--0.5200813 b 0
-
-\2-grams:
--0.2807584 <s> a
--0.3222944 <s> b
--0.70763 a </s>
--0.6780174 a a
--0.2261675 a b
--0.7991036 b </s>
--0.1237299 b a
--1.050158 b b
-
-\end\
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/toy_data/toy1.lm
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/toy_data/toy1.lm b/ext/kenlm/lm/interpolate/toy_data/toy1.lm
deleted file mode 100644
index fba4263..0000000
--- a/ext/kenlm/lm/interpolate/toy_data/toy1.lm
+++ /dev/null
@@ -1,23 +0,0 @@
-
-\data\
-ngram 1=5
-ngram 2=8
-
-\1-grams:
--1.3010299957 </s>
--99 <s> -7.446389
--99 <unk>
--0.6020599913 a 0
--0.6020599913 b 0
-
-\2-grams:
--0.15490196 <s> a
--0.5228787453 <s> b
--1 a </s>
--1.5228787453 a a
--0.0604807474 a b
--0.5228787453 b </s>
--0.3010299957 b a
--0.6989700043 b b
-
-\end\
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/toy_data/toy2.lm
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/toy_data/toy2.lm b/ext/kenlm/lm/interpolate/toy_data/toy2.lm
deleted file mode 100644
index 67c2ba1..0000000
--- a/ext/kenlm/lm/interpolate/toy_data/toy2.lm
+++ /dev/null
@@ -1,23 +0,0 @@
-
-\data\
-ngram 1=5
-ngram 2=8
-
-\1-grams:
--0.3979400087 </s>
--99 <s> -7.446389
--99 <unk>
--0.3979400087 a 0
--0.6989700043 b 0
-
-\2-grams:
--0.3979400087 <s> a
--0.2218487496 <s> b
--0.6989700043 a </s>
--0.3010299957 a a
--0.5228787453 a b
--1 b </s>
--0.0222763947 b a
--1.3010299957 b b
-
-\end\
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/train_params_grant_main.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/train_params_grant_main.cc b/ext/kenlm/lm/interpolate/train_params_grant_main.cc
deleted file mode 100644
index a106cac..0000000
--- a/ext/kenlm/lm/interpolate/train_params_grant_main.cc
+++ /dev/null
@@ -1,561 +0,0 @@
-#include "lm/ngram_query.hh"
-#include "lm/model.hh"
-#include "lm/word_index.hh"
-#include "lm/interpolate/enumerate_global_vocab.hh"
-
-#include <string>
-#include <vector>
-#include <iostream>
-#include <fstream>
-#include <map>
-#include <iomanip>
-
-#include <boost/program_options.hpp>
-#include <boost/version.hpp>
-#include <boost/foreach.hpp>
-
-#include "util/fixed_array.hh"
-
-#include <Eigen/Eigen>
-#include <Eigen/Dense>
-
-// typedef Eigen::MatrixXf FMatrix;
-// typedef Eigen::VectorXf FVector;
-typedef Eigen::MatrixXd DMatrix;
-typedef Eigen::VectorXd DVector;
-
-bool HAS_BIAS = true;
-
-using namespace lm::ngram;
-using namespace lm;
-
-inline double logProb(Model *model, const std::vector<std::string> &ctx,
- WordIndex word_idx) {
- // Horribly inefficient
- const Vocabulary &vocab = model->GetVocabulary();
-
- State nextState; // throwaway
-
- WordIndex context_idx[ctx.size()];
-
- // reverse context
- for (std::size_t i = 0; i < ctx.size(); i++) {
- context_idx[ctx.size() - 1 - i] = vocab.Index(ctx[i]);
- }
-
- FullScoreReturn score = model->FullScoreForgotState(
- context_idx, &(context_idx[ctx.size() - 1]), word_idx, nextState);
-
- double ret = score.prob;
- // std::cerr << "w: " << word << " p: " << ret << std::endl;
- return ret;
-}
-
-inline double logProb(Model *model, double unkprob,
- const std::vector<std::string> &ctx,
- const std::string &word) {
- // Horribly inefficient
- const Vocabulary &vocab = model->GetVocabulary();
-
- WordIndex word_idx = vocab.Index(word);
- if (word_idx == lm::kUNK) return unkprob;
-
- return logProb(model, ctx, word_idx);
-}
-
-void set_features(const std::vector<std::string> &ctx, const std::string &word,
- const std::vector<Model *> &models,
- const std::vector<double> &unkprobs, DVector &v) {
- if (HAS_BIAS) {
- v(0) = 1;
- for (std::size_t i = 0; i < models.size(); ++i)
- v(i + 1) = logProb(models[i], unkprobs[i], ctx, word);
- } else {
- for (std::size_t i = 0; i < models.size(); ++i)
- v(i) = logProb(models[i], unkprobs[i], ctx, word);
- }
-}
-
-void train_params(const std::vector<std::vector<std::string> > &corpus,
- const std::vector<std::string> &vocab,
- const std::vector<Model *> &models) {
- using namespace std;
-
- // A safeguarded Newton's method to find optimum parameters.
- // Reverts to steepest-descent linesearch if Newton step does not improve
- // objective.
- //
- // Two Boolean variables below are used to "AllowExtrapolation" and
- // "AllowNegativeParams".
-
- bool AllowExtrapolation = true; // if true, params need not sum to one
- bool AllowNegativeParams = true; // if true, params can be negative
- const int ITERATIONS = 20;
- double minstepsize = 1.0e-9; // convergence criterion
- int context_size = 5; // (context_size+1)-grams considered in perplexity
- double stepdecreasefactor = 0.1; // if step unsuccessful
- double initstepsize = 1.0; // Initial step size
- std::size_t linesinstartercorpus = 12; // The first few lines are tuned
- // first, to find basin of attraction
- // for Newton
- // bias + #models
- const std::size_t nlambdas = models.size() + (HAS_BIAS ? 1 : 0);
-
- // initialize to sum to 1
- DVector params = DVector::Constant(nlambdas, 1.0 / nlambdas);
- DMatrix N = DMatrix::Constant(
- nlambdas, nlambdas - 1,
- -1.0 / sqrt((nlambdas - 1) * (nlambdas - 1) + nlambdas - 1.0));
- for (unsigned i = 0; i < nlambdas - 1; ++i)
- N(i, i) = N(i, i) * (1.0 - nlambdas);
- // N is nullspace matrix, each column sums to zero
-
- cerr << setprecision(16) << "++ Parameter training ++" << endl;
- if (AllowExtrapolation)
- cerr << " Allowing extrapolation (sharpening and flattening of individual "
- "LM distributions)" << endl;
- else
- cerr << " Interpolating only (not sharpening or flattening individual LM "
- "distributions)" << endl;
- if (AllowNegativeParams)
- cerr << " Allowing negative parameters\n"
- << " (more general but slow and rarely useful\n"
- << " -LM with negative weight has probability rankings reversed and "
- "is weighted more highly than all LMs with positive weights)"
- << endl;
- else
- cerr << "Not allowing negative parameters (mild assumption, and faster)"
- << endl;
- cerr << " Maximum number of iterations: " << ITERATIONS << endl;
- cerr << " Minimum step size: " << minstepsize << endl;
- cerr << " Perplexity computed with " << context_size + 1 << "-grams" << endl;
-
- if ((!AllowExtrapolation) && (nlambdas == 1)) {
- // Nothing to optimize. One parameter, and it sums to one.
- cerr << "Training complete. Best weights:" << endl;
- cerr << setprecision(16) << 1.0 << endl;
- return;
- }
-
- // Smart initialization of full tuning by tuning on smaller set first
- vector<std::size_t> linestotune;
- if (linesinstartercorpus < corpus.size())
- linestotune.push_back(linesinstartercorpus);
- linestotune.push_back(corpus.size());
-
- for (std::size_t setiter = 0; setiter < linestotune.size(); ++setiter) {
- cerr << " Now tuning the first " << linestotune[setiter] << " lines"
- << endl;
-
- vector<DVector> paramhistory;
- double bestppl = 0.0; // best recorded ppl
- DVector bestgrad = DVector::Zero(nlambdas); // corresp. gradient,
- // feasible direction
- DVector bestparams = DVector::Zero(nlambdas); // corresp. weights
- double maxbestgradstep = 0.0; // max feasible step in grad. direction
- double stepsize = initstepsize; // Initial step size
-
- for (int iter = 0; iter < ITERATIONS; ++iter) { // iterations
- cerr << "ITERATION " << iter + 1 << " (of max " << ITERATIONS
- << "), step size " << stepsize << " (of min " << minstepsize
- << "), weights: " << endl;
- cerr << params << endl;
-
- paramhistory.push_back(params);
- // Hard-coded to be 6-gram perplexity
- vector<string> context(context_size, "<s>");
- double ppl = 0.0;
- DVector grad = DVector::Zero(nlambdas);
- DMatrix H = DMatrix::Zero(nlambdas, nlambdas);
- cerr << "o";
-
- std::vector<double> unkprobs(models.size());
- // for each sentence in tuning corpus
- for (std::size_t ci = 0; ci < linestotune[setiter]; ++ci) {
- const vector<string> &sentence = corpus[ci];
- // pad our beginning context
- std::fill(context.begin(), context.end(), "<s>");
-
- // for each word in sentence
- for (std::size_t t = 0; t < sentence.size(); ++t) {
- // fill in unk probabilities for this context, to avoid having to
- // look them up redundantly later
- for (std::size_t mi = 0; mi < models.size(); ++mi) {
- unkprobs[mi] = logProb(models[mi], context, lm::kUNK);
- }
-
- DVector feats = DVector::Zero(nlambdas);
- // probs for actual n-gram
- set_features(context, sentence[t], models, unkprobs, feats);
-
- double z = 0.0;
- double maxlogprob = 0.0; // Allows us to avoid overflow with
- // negative params
- DVector expectfeats = DVector::Zero(nlambdas);
- DMatrix expectfeatmatrix = DMatrix::Zero(nlambdas, nlambdas);
- // Logically, this should be in the loop's scope
- DVector iterfeats = DVector::Zero(nlambdas);
-
- // probs over possible n-grams, for normalization
- for (std::size_t i = 0; i < vocab.size(); ++i) {
- set_features(context, vocab[i], models, unkprobs, iterfeats);
- double logprob = params.dot(iterfeats);
- if (i == 0) {
- // maxlogprob=logprob;// more precise, less underflow
- maxlogprob = 0.0; // reduces number of updates
- } else if (logprob > maxlogprob) {
- // Adjust all old values to new scaling
- double adjust = exp(maxlogprob - logprob);
- z *= adjust;
- expectfeats *= adjust;
- expectfeatmatrix *= adjust;
- maxlogprob = logprob;
- }
- double us = exp(params.dot(iterfeats) - maxlogprob); // measure
-
- z += us;
- expectfeats += us * iterfeats;
- expectfeatmatrix += us * (iterfeats * iterfeats.transpose());
- }
- expectfeats /= z; // Expectation
- expectfeatmatrix /= z; // Expectation
-
- // Add sentence[t] to the end of the context
- context[0] = sentence[t];
- std::rotate(context.begin(), context.begin() + 1, context.end());
-
- // Perplexity (actually log(perplexity))
- ppl += params.dot(feats) - log(z);
- // Gradient
- grad += feats - expectfeats;
- // Hessian
- H += -expectfeatmatrix + expectfeats * expectfeats.transpose();
- }
- cerr << ".";
- }
- ppl *= -1.0 / corpus.size();
- // The gradient and Hessian coefficients cancel out, so don't really need
- // to do this, but it's fast.
- grad *= -1.0 / corpus.size();
- H *= -1.0 / corpus.size();
- cerr << " log(PPL)=" << ppl << " PPL=" << exp(ppl) << endl;
-
- // Use results to determine next params to evaluate
- if ((ppl < bestppl) || (iter == 0)) {
- // Found a new best
- bestppl = ppl;
- bestparams = params;
- double beststepsize = stepsize;
- if (iter > 0)
- cerr << " New best point found, step size " << beststepsize << endl;
- else
- cerr << " New best point found" << endl;
-
- bestgrad = grad;
- DVector deltaparams = DVector::Zero(nlambdas);
-
- bool reverttograd = false;
-
- {
- double gradnorm = 0.0;
- double solvenorm = 0.0;
- double errnorm = 0.0;
- // Find Newton step
- if (AllowExtrapolation) {
- deltaparams = -H.colPivHouseholderQr().solve(grad);
- Eigen::SelfAdjointEigenSolver<DMatrix> eigensolver(H);
- cerr << "Eigenvalues (negative values should be negligible):\n"
- << eigensolver.eigenvalues() << endl;
- gradnorm = grad.norm();
- solvenorm = (H * deltaparams).norm();
- errnorm = (grad + H * deltaparams).norm();
- } else {
- // Project gradient to interpolation space
- bestgrad = N * N.transpose() * bestgrad;
-
- // need to work in nullspace to maintain unit sum
- DMatrix Hnull = DMatrix::Zero(nlambdas - 1, nlambdas - 1);
-
- // Looks like we don't need the three lines below -- we can do it
- // in-line (if we don't want eigenvalues)
- Hnull = N.transpose() * H * N;
- Eigen::SelfAdjointEigenSolver<DMatrix> eigensolver(Hnull);
- cerr << "Eigenvalues (best if all positive):\n"
- << eigensolver.eigenvalues() << endl;
- deltaparams =
- -N * Hnull.fullPivHouseholderQr().solve(N.transpose() * grad);
- gradnorm = (N.transpose() * grad).norm();
- solvenorm = (Hnull * deltaparams).norm();
- errnorm = (N.transpose() * grad + Hnull * deltaparams).norm();
- }
- // eventually, params = bestparams + deltaparams;
- cerr << " Error norm " << errnorm << ", gradient norm " << gradnorm
- << ", solution norm " << solvenorm << endl;
- // Check for numerical errors. Don't trust Newton step if they are too
- // big.
- if (errnorm < 1e-12 * std::max(1.0, std::min(gradnorm, solvenorm))) {
- stepsize = 0.0;
- for (std::size_t i = 0; i < nlambdas; i++)
- stepsize += deltaparams(i) * deltaparams(i);
- stepsize = sqrt(stepsize); // holds length of Newton step
- cerr << "Newton step, length " << stepsize << ": " << endl;
- cerr << deltaparams << endl;
-
- // Don't let the Newton step get much bigger than last successful
- // step (likely would have to shrink later, anyway)
- if (stepsize > 2.0 * beststepsize) {
- stepsize = 1.5 * beststepsize;
- reverttograd = true;
- cerr << "Reverting to gradient, because Newton step is too large."
- << endl;
- }
- } else {
- stepsize = 1.5 * beststepsize;
- reverttograd = true;
- cerr << "Reverting to gradient, because Newton step computation "
- "unsuccessful." << endl;
- }
- // Make the gradient unit norm, in feasible search direction.
- if (!AllowNegativeParams) {
- // Project gradient to be a feasible search direction
- vector<bool> active(nlambdas, false);
- std::size_t numactive = 0;
- for (std::size_t i = 0; i < nlambdas; i++) {
- // Project gradient to inactive constraints
- if ((bestparams(i) == 0) && (bestgrad(i) > 0)) {
- active[i] = true;
- bestgrad(i) = 0.0; // Do this now, in case extrapolation
- // allowed.
- ++numactive;
- }
- }
- if (numactive > 0) {
- if (!AllowExtrapolation) {
- // Project gradient, for activity concerns
- DMatrix tmpN = DMatrix::Constant(
- nlambdas, nlambdas - 1,
- -1.0 / sqrt((nlambdas - numactive - 1) *
- (nlambdas - numactive - 1) +
- nlambdas - numactive - 1.0));
-
- for (std::size_t i = 0; i < nlambdas - 1; ++i)
- tmpN(i, i) = tmpN(i, i) * (1.0 - (nlambdas - numactive));
-
- for (std::size_t i = 0; i < nlambdas; ++i) {
- if (active[i]) {
- for (std::size_t j = 0; j < nlambdas - 1; ++i) {
- tmpN(i, j) = 0;
- }
- }
- }
-
- // projected gradient onto unit sum and active set constraints
- bestgrad = -tmpN * tmpN.transpose() * bestgrad;
- }
- }
- }
- }
- double norm = 0.0;
- for (std::size_t i = 0; i < nlambdas; i++)
- norm += bestgrad(i) * bestgrad(i);
- if (norm != 0) {
- bestgrad /= sqrt(norm);
- } else {
- cerr << " Gradient is zero. Exiting.";
- break;
- }
- cerr << "Gradient, unit length: " << endl;
- cerr << bestgrad << endl;
-
- // Find max step in gradient direction that remains feasible.
- if (!AllowNegativeParams) {
- double limitfraction = 0.5; // Not 1: If Newton step is bad, probably
- // will need to reduce later anyway
- for (std::size_t i = 0; i < nlambdas; i++) {
- if (bestparams(i) - maxbestgradstep * bestgrad(i) < 0) {
- double tmplimitfraction =
- bestparams(i) / (bestgrad(i) * maxbestgradstep);
- if (tmplimitfraction < limitfraction)
- limitfraction = tmplimitfraction;
- }
- }
- maxbestgradstep = stepsize * limitfraction;
- cerr << " Max grad step: " << maxbestgradstep << endl;
- } else {
- maxbestgradstep = stepsize;
- }
-
- if (!reverttograd) {
- if (!AllowNegativeParams) {
- for (std::size_t i = 0; i < nlambdas; i++) {
- if (bestparams(i) + deltaparams(i) < 0) {
- // Can't do Newton step. Revert to descent.
- reverttograd = true;
- }
- }
- }
- if (reverttograd) {
- cerr << "Reverting to gradient, since Newton step infeasible:"
- << endl;
- }
- }
-
- if (reverttograd) {
- stepsize = maxbestgradstep;
- deltaparams = -bestgrad * stepsize;
- }
-
- params = bestparams + deltaparams;
- cerr << "Change in weights from best, step size " << stepsize << ": "
- << endl;
- cerr << deltaparams << endl;
- } else {
- // Last attempt failed at being better, so move in gradient direction
- // with reduced step.
- // stepsize reduction factor is empirical
- stepsize = std::min(stepdecreasefactor * stepsize, maxbestgradstep);
- cerr << "Taking smaller step: " << stepsize << endl;
- params = bestparams - bestgrad * stepsize;
- }
- // Clean the parameters up.
- double sumparams = 0.0;
- for (std::size_t i = 0; i < nlambdas; i++) {
- if (!AllowNegativeParams) {
- if (params(i) < 1e-12) {
- // snap to zero, for active set and duplicate weights
- params(i) = 0;
- }
- }
- sumparams += params(i);
- }
- if (!AllowExtrapolation) params /= sumparams;
-
- bool duplicateentry = false;
- for (std::size_t i = 0; i < paramhistory.size(); ++i) {
- if (params == paramhistory[i]) duplicateentry = true;
- }
- while ((duplicateentry) && (stepsize >= minstepsize)) {
- cerr << "Duplicate weight found: " << endl;
- cerr << params << endl;
- stepsize *= 0.5; // Step in this direction is duplicate, so try again
- // with smaller step
- params = bestparams - stepsize * bestgrad;
-
- sumparams = 0.0;
- for (std::size_t i = 0; i < nlambdas; i++) {
- if (!AllowNegativeParams) {
- if (params(i) < 1e-12) params(i) = 0;
- }
- sumparams += params(i);
- }
- if (!AllowExtrapolation) params /= sumparams;
-
- duplicateentry = false;
- for (std::size_t i = 0; i < paramhistory.size(); ++i) {
- if (params == paramhistory[i]) duplicateentry = true;
- }
- }
- if (stepsize < minstepsize) break; // No need to make another step
- }
-
- params = bestparams; // So that next setiter is correct
- cerr << "Training complete. Best weights:" << endl;
- cerr << params << endl;
- }
-}
-
-int main(int argc, char **argv) {
- std::string tuning_data;
- std::vector<std::string> lms;
-
- try {
- namespace po = boost::program_options;
- po::options_description options("train-params");
-
- options.add_options()("help,h", po::bool_switch(),
- "Show this help message")(
- "no_bias_term,B", po::bool_switch(), "Do not include a 'bias' feature")(
- "tuning_data,t", po::value<std::string>(&tuning_data),
- "File to tune perplexity on")(
- "model,m", po::value<std::vector<std::string> >(&lms),
- "Language models in KenLM format to interpolate");
- po::variables_map vm;
- po::store(po::parse_command_line(argc, argv, options), vm);
-
- // Display help
- if (argc == 1 || vm["help"].as<bool>()) {
- std::cerr << options << std::endl;
- return 1;
- }
- if (vm["no_bias_term"].as<bool>()) HAS_BIAS = false;
- lms = vm["model"].as<std::vector<std::string> >();
- tuning_data = vm["tuning_data"].as<std::string>();
- } catch (const std::exception &e) {
- std::cerr << e.what() << std::endl;
- return 1;
- }
- if (lms.size() < 2) {
- std::cerr
- << "Please specify at least two language model files with -m LM.KLM\n";
- return 1;
- }
- if (tuning_data.empty()) {
- std::cerr << "Please specify tuning set with -t FILE.TXT\n";
- return 1;
- }
-
- // Growable vocab here
- // GrowableVocab gvoc(100000); //dummy default
-
- // no comment
- std::map<std::string, int *> vmap;
-
- // stuff it into the
- EnumerateGlobalVocab *globalVocabBuilder =
- new EnumerateGlobalVocab(&vmap, lms.size());
-
- Config cfg;
- cfg.enumerate_vocab = (EnumerateVocab *)globalVocabBuilder;
-
- // load models
- std::vector<Model *> models;
- for (std::size_t i = 0; i < lms.size(); i++) {
- std::cerr << "Loading LM file: " << lms[i] << std::endl;
-
- // haaaack
- globalVocabBuilder->SetCurModel(i); // yes this is dumb
-
- Model *this_model = new Model(lms[i].c_str(), cfg);
- models.push_back(this_model);
- }
-
- // assemble vocabulary vector
- std::vector<std::string> vocab;
- std::cerr << "Global Vocab Map has size: " << vmap.size() << std::endl;
-
- for (std::map<std::string, int *>::iterator iter = vmap.begin();
- iter != vmap.end(); ++iter) {
- vocab.push_back(iter->first);
- }
- std::cerr << "Vocab vector has size: " << vocab.size() << std::endl;
-
- // load context sorted ngrams into vector of vectors
- std::vector<std::vector<std::string> > corpus;
-
- std::cerr << "Loading context-sorted ngrams: " << tuning_data << std::endl;
- std::ifstream infile(tuning_data.c_str());
-
- for (std::string line; std::getline(infile, line);) {
- std::vector<std::string> words;
- std::stringstream stream(line);
- std::string word;
-
- while (stream >> word)
- words.push_back(word);
- corpus.push_back(words);
- }
-
- train_params(corpus, vocab, models);
-
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/train_params_main.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/train_params_main.cc b/ext/kenlm/lm/interpolate/train_params_main.cc
deleted file mode 100644
index 39233e8..0000000
--- a/ext/kenlm/lm/interpolate/train_params_main.cc
+++ /dev/null
@@ -1,349 +0,0 @@
-#include "lm/ngram_query.hh"
-#include "lm/model.hh"
-#include "lm/word_index.hh"
-#include "lm/interpolate/enumerate_global_vocab.hh"
-
-
-#include <string>
-#include <vector>
-#include <iostream>
-#include <fstream>
-#include <map>
-
-#include <boost/program_options.hpp>
-#include <boost/version.hpp>
-#include <boost/foreach.hpp>
-
-#include "util/fixed_array.hh"
-
-#include <Eigen/Eigen>
-
-typedef Eigen::MatrixXf FMatrix;
-typedef Eigen::VectorXf FVector;
-
-bool HAS_BIAS = true;
-
-using namespace lm::ngram;
-using namespace lm;
-
-inline float logProb(Model * model, const std::vector<std::string>& ctx, const std::string& word) {
-
- // Horribly inefficient
- const Vocabulary &vocab = model->GetVocabulary();
-
- State nextState; //throwaway
-
- WordIndex word_idx = vocab.Index(word);
- WordIndex context_idx[ctx.size()];
-
- //reverse context
- for(unsigned int i = 0; i < ctx.size(); i++) {
- context_idx[ctx.size() - 1 - i] = vocab.Index(ctx[i]);
- }
-
- FullScoreReturn score = model->FullScoreForgotState(context_idx, &(context_idx[ctx.size() -1]), word_idx, nextState);
-
- float ret = score.prob;
- //std::cerr << "w: " << word << " p: " << ret << std::endl;
- return ret;
-}
-
-void set_features(const std::vector<std::string>& ctx,
- const std::string& word,
- const std::vector<Model *>& models,
- FVector& v) {
-
- //std::cerr << "setting feats for " << word << std::endl;
-
- if (HAS_BIAS) {
- v(0) = 1;
- for (unsigned i=0; i < models.size(); ++i)
- v(i + 1) = logProb(models[i], ctx, word);
- } else {
- for (unsigned i=0; i < models.size(); ++i)
- v(i) = logProb(models[i], ctx, word);
- }
-}
-
-void translate_input(
- const std::vector<std::vector<std::string> >& corpus,
- const std::vector<std::string>& gvocab,
- const std::vector<Model *>& models,
- std::vector<std::vector<std::vector<WordIndex> > >&translated_corpus,
- std::vector<std::vector<WordIndex> >&translated_vocab
- ) {
- translated_corpus.resize(models.size());
- translated_vocab.resize(models.size());
- for (unsigned mn=0; mn < models.size(); ++mn) { // models
-
- const Vocabulary &vocab = models[mn]->GetVocabulary();
-
- for (unsigned i = 0; i < gvocab.size(); ++i) {
- translated_vocab[mn].push_back(vocab.Index(gvocab[i]));
- }
-
- translated_corpus[mn].resize(corpus.size());
- for (unsigned ci = 0; ci < corpus.size(); ++ci) { // sentences in tuning corpus
- const std::vector<std::string>& sentence = corpus[ci];
- for (int t = sentence.size() -1; t >= 0; --t) { // words in sentence
- translated_corpus[mn][ci].push_back(vocab.Index(sentence[t]));
- }
- for (int i=0; i<5; ++i) {
- translated_corpus[mn][ci].push_back(vocab.Index("<s>"));
- }
- }
- }
-}
-
-
-void train_params_fast(
- const std::vector<std::vector<std::string> >& corpus,
- const std::vector<std::string>& vocab,
- const std::vector<Model *>& models) {
- using namespace std;
-
- // model / sentence / words in sentence in reverse order with <s> padding
- std::vector<std::vector<std::vector<WordIndex> > > t_corpus;
- std::vector<std::vector<WordIndex> > t_vocab;
- translate_input(corpus, vocab, models, t_corpus, t_vocab);
-
-
-
- const int ITERATIONS = 10;
- const int nlambdas = models.size() + (HAS_BIAS ? 1 : 0); // bias + #models
- FVector params = FVector::Zero(nlambdas);
- vector<FVector> feats(vocab.size(), params);
- vector<float> us(vocab.size(), 0);
- vector<float> ps(vocab.size(), 0);
- FVector grad = FVector::Zero(nlambdas);
- FMatrix H = FMatrix::Zero(nlambdas, nlambdas);
- FVector ef = FVector::Zero(nlambdas);
- for (int iter = 0; iter < ITERATIONS; ++iter) { // iterations
- grad.setZero();
- H.setZero();
- double loss = 0;
- unsigned numchars = 0;
- for (unsigned ci = 0; ci < corpus.size(); ++ci) { // sentences in tuning corpus
- const vector<string>& sentence = corpus[ci];
- double z = 0;
- for (int t = sentence.size() -1 ; t >=0; --t) { // words in sentence
- ++numchars;
- int ref_word = 0;
- for (unsigned i = 0; i < vocab.size(); ++i) { // vocab
- // set_features(context, vocab[i], models, feats[i]);
- for (unsigned j=0; j < models.size(); ++j) {
- // NOTE: reference ---- WordIndex word_idx = t_corpus[j][ci][t];
- WordIndex word_idx = t_vocab[j][i];
- State nextState; //throwaway
- FullScoreReturn score = models[j]->FullScoreForgotState(&(t_corpus[j][ci][t]), &(t_corpus[j][ci][t+5]), word_idx, nextState);
- feats[i](j) = score.prob;
- // feats[i](j) = logProb(models[j], ctx, word);
- }
-
- us[i] = params.dot(feats[i]);
- z += exp(double(us[i]));
- }
- //std::cerr << "there..." << std::endl;
- const float logz = log(z);
-
- // expected feature values
- ef.setZero();
- for (unsigned i = 0; i < vocab.size(); ++i) {
- ps[i] = expf(us[i] - logz);
- ef += ps[i] * feats[i];
- }
- loss -= log(ps[ref_word]);
- const FVector& reffeats = feats[ref_word];
- grad += ef - reffeats;
-
- // Hessian
- for (unsigned i = 0; i < vocab.size(); ++i)
- H.noalias() += ps[i] * feats[i] * feats[i].transpose() -
- ps[i] * feats[i] * ef.transpose();
-
- // this should just be the state for each model
- }
- cerr << ".";
- }
- cerr << "ITERATION " << (iter + 1) << ": PPL=" << exp(loss / numchars) << endl;
- params = H.colPivHouseholderQr().solve(grad);
- cerr << params << endl;
- }
-}
-
-
-
-
-//const util::FixedArray<Model *>& models)
-void train_params(
- const std::vector<std::vector<std::string> >& corpus,
- const std::vector<std::string>& vocab,
- const std::vector<Model *>& models) {
- using namespace std;
-
- vector<string> context(5, "<s>");
- const int ITERATIONS = 10;
- const int nlambdas = models.size() + (HAS_BIAS ? 1 : 0); // bias + #models
- FVector params = FVector::Zero(nlambdas);
- vector<FVector> feats(vocab.size(), params);
- vector<float> us(vocab.size(), 0);
- vector<float> ps(vocab.size(), 0);
- FVector grad = FVector::Zero(nlambdas);
- FMatrix H = FMatrix::Zero(nlambdas, nlambdas);
- FVector ef = FVector::Zero(nlambdas);
- for (int iter = 0; iter < ITERATIONS; ++iter) { // iterations
- grad.setZero();
- H.setZero();
- double loss = 0;
- unsigned numchars = 0;
- for (unsigned ci = 0; ci < corpus.size(); ++ci) { // sentences in tuning corpus
- const vector<string>& sentence = corpus[ci];
- std::fill(context.begin(), context.end(), "<s>");
- for (unsigned t = 0; t < sentence.size(); ++t) { // words in sentence
- ++numchars;
- const string& ref_word_string = sentence[t];
- int ref_word = 0; // TODO
- double z = 0;
- //std::cerr << "here..." << std::endl;
- for (unsigned i = 0; i < vocab.size(); ++i) { // vocab
- set_features(context, vocab[i], models, feats[i]);
- us[i] = params.dot(feats[i]);
- z += exp(double(us[i]));
- }
- //std::cerr << "there..." << std::endl;
- context.push_back(ref_word_string);
- const float logz = log(z);
-
- // expected feature values
- ef.setZero();
- for (unsigned i = 0; i < vocab.size(); ++i) {
- ps[i] = expf(us[i] - logz);
- ef += ps[i] * feats[i];
- }
- loss -= log(ps[ref_word]);
- const FVector& reffeats = feats[ref_word];
- grad += ef - reffeats;
-
- // Hessian
- for (unsigned i = 0; i < vocab.size(); ++i)
- H.noalias() += ps[i] * feats[i] * feats[i].transpose() -
- ps[i] * feats[i] * ef.transpose();
-
- // this should just be the state for each model
- }
- cerr << ".";
- }
- cerr << "ITERATION " << (iter + 1) << ": PPL=" << exp(loss / numchars) << endl;
- params = H.colPivHouseholderQr().solve(grad);
- cerr << params << endl;
- }
-}
-
-int main(int argc, char** argv) {
-
- std::string tuning_data;
- std::vector<std::string> lms;
-
- try {
- namespace po = boost::program_options;
- po::options_description options("train-params");
-
- options.add_options()
- ("help,h", po::bool_switch(), "Show this help message")
- ("no_bias_term,B", po::bool_switch(), "Do not include a 'bias' feature")
- ("tuning_data,t", po::value<std::string>(&tuning_data), "File to tune perplexity on")
- ("model,m", po::value<std::vector<std::string> >(&lms), "Language models in KenLM format to interpolate");
- po::variables_map vm;
- po::store(po::parse_command_line(argc, argv, options), vm);
-
- // Display help
- if(argc == 1 || vm["help"].as<bool>()) {
- std::cerr << options << std::endl;
- return 1;
- }
- if (vm["no_bias_term"].as<bool>())
- HAS_BIAS = false;
- lms = vm["model"].as<std::vector<std::string> >();
- tuning_data = vm["tuning_data"].as<std::string>();
- }
- catch(const std::exception &e) {
-
- std::cerr << e.what() << std::endl;
- return 1;
-
- }
- if (lms.size() < 2) {
- std::cerr << "Please specify at least two language model files with -m LM.KLM\n";
- return 1;
- }
- if (tuning_data.empty()) {
- std::cerr << "Please specify tuning set with -t FILE.TXT\n";
- return 1;
- }
-
- //Growable vocab here
- //GrowableVocab gvoc(100000); //dummy default
-
- std::map<std::string, int*> vmap;
- util::FixedArray<WordIndex> vm(2);
-
- //stuff it into the
- EnumerateGlobalVocab * globalVocabBuilder = new EnumerateGlobalVocab(&vmap, lms.size());
- // EnumerateGlobalVocab * globalVocabBuilder = new EnumerateGlobalVocab(vm);
-
- Config cfg;
- cfg.enumerate_vocab = (EnumerateVocab *) globalVocabBuilder;
-
- //load models
- //util::FixedArray<Model *> models(lms.size());
- std::vector<Model *> models;
- for(int i=0; i < lms.size(); i++) {
- std::cerr << "Loading LM file: " << lms[i] << std::endl;
-
- //haaaack
- globalVocabBuilder->SetCurModel(i); //yes this is dumb
-
- //models[i] = new Model(lms[i].c_str());
- Model * this_model = new Model(lms[i].c_str(), cfg);
- models.push_back( this_model );
-
- }
-
- //assemble vocabulary vector
- std::vector<std::string> vocab;
- std::cerr << "Global Vocab Map has size: " << vmap.size() << std::endl;
-
- std::pair<StringPiece,int *> me;
-
- for(std::map<std::string, int*>::iterator iter = vmap.begin(); iter != vmap.end(); ++iter) {
- vocab.push_back(iter->first);
- }
- std::cerr << "Vocab vector has size: " << vocab.size() << std::endl;
-
- //load context sorted ngrams into vector of vectors
- std::vector<std::vector<std::string> > corpus;
-
- std::cerr << "Loading context-sorted ngrams: " << tuning_data << std::endl;
- std::ifstream infile(tuning_data);
-
- for(std::string line; std::getline(infile, line); ) {
-
- std::vector<std::string> words; {
-
- std::stringstream stream(line);
- std::string word;
-
- while(stream >> word) {
- words.push_back(word);
- }
- }
- corpus.push_back(words);
- }
-
- train_params_fast(corpus, vocab, models);
-
- return 0;
-}
-
-
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/tune_derivatives.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/tune_derivatives.cc b/ext/kenlm/lm/interpolate/tune_derivatives.cc
deleted file mode 100644
index c20e637..0000000
--- a/ext/kenlm/lm/interpolate/tune_derivatives.cc
+++ /dev/null
@@ -1,91 +0,0 @@
-#include "lm/interpolate/tune_derivatives.hh"
-
-namespace lm { namespace interpolate {
-
-ComputeDerivative::ComputeDerivative(const util::FixedArray<Instance> &instances, const Matrix &ln_unigrams, WordIndex bos)
- : instances_(instances), ln_unigrams_(ln_unigrams), bos_(bos) {
- neg_correct_summed_ = Vector::Zero(ln_unigrams.cols());
- for (const Instance *i = instances.begin(); i != instances.end(); ++i) {
- neg_correct_summed_ -= i->ln_correct;
- }
-}
-
-Accum ComputeDerivative::Iteration(const Vector &weights, Vector &gradient, Matrix &hessian) {
- gradient = neg_correct_summed_;
- hessian = Matrix::Zero(weights.rows(), weights.rows());
-
- // TODO: loop instead to force low-memory evaluation
- // Compute p_I(x).
- Vector interp_uni((ln_unigrams_ * weights).array().exp());
- // Even -inf doesn't work for <s> because weights can be negative. Manually set it to zero.
- interp_uni(bos_) = 0.0;
- Accum Z_epsilon = interp_uni.sum();
- interp_uni /= Z_epsilon;
- // unigram_cross(i) = \sum_{all x} p_I(x) ln p_i(x)
- Vector unigram_cross(ln_unigrams_.transpose() * interp_uni);
-
- Accum sum_B_I = 0.0;
- Accum sum_ln_Z_context = 0.0;
-
- Vector weighted_extensions;
- Matrix convolve;
- Vector full_cross;
-
- for (const Instance *n = instances_.begin(); n != instances_.end(); ++n) {
- Accum ln_weighted_backoffs = n->ln_backoff.dot(weights);
- Accum weighted_backoffs = exp(ln_weighted_backoffs);
-
- // Compute \sum_{x: model does not backoff to unigram} p_I(x)
- Accum sum_x_p_I = 0.0;
- for (std::vector<WordIndex>::const_iterator x = n->extension_words.begin(); x != n->extension_words.end(); ++x) {
- sum_x_p_I += interp_uni(*x);
- }
- weighted_extensions = (n->ln_extensions * weights).array().exp();
- Accum Z_context = Z_epsilon * weighted_backoffs * (1.0 - sum_x_p_I) + weighted_extensions.sum();
- sum_ln_Z_context += log(Z_context);
-
- Accum B_I = Z_epsilon / Z_context * weighted_backoffs;
- sum_B_I += B_I;
-
- // This is the gradient term for this instance except for -log p_i(w_n | w_1^{n-1}) which was accounted for as part of neg_correct_sum_.
- // full_cross(i) is \sum_{all x} p_I(x | context) log p_i(x | context)
- full_cross =
- // Uncorrected term
- B_I * (n->ln_backoff + unigram_cross)
- // Correction term: add correct values
- + n->ln_extensions.transpose() * weighted_extensions / Z_context
- // Subtract values that should not have been charged.
- - sum_x_p_I * B_I * n->ln_backoff;
- for (std::vector<WordIndex>::const_iterator x = n->extension_words.begin(); x != n->extension_words.end(); ++x) {
- full_cross.noalias() -= interp_uni(*x) * B_I * ln_unigrams_.row(*x);
- }
-
- gradient += full_cross;
-
- convolve = unigram_cross * n->ln_backoff.transpose();
- // There's one missing term here, which is independent of context and done at the end.
- hessian.noalias() +=
- // First term of Hessian, assuming all models back off to unigram.
- B_I * (convolve + convolve.transpose() + n->ln_backoff * n->ln_backoff.transpose())
- // Second term of Hessian, with correct full probabilities.
- - full_cross * full_cross.transpose();
-
- // Adjust the first term of the Hessian to account for extension
- for (std::size_t x = 0; x < n->extension_words.size(); ++x) {
- WordIndex universal_x = n->extension_words[x];
- hessian.noalias() +=
- // Replacement terms.
- weighted_extensions(x) / Z_context * n->ln_extensions.row(x).transpose() * n->ln_extensions.row(x)
- // Presumed unigrams. TODO: individual terms with backoffs pulled out? Maybe faster?
- - interp_uni(universal_x) * B_I * (ln_unigrams_.row(universal_x).transpose() + n->ln_backoff) * (ln_unigrams_.row(universal_x) + n->ln_backoff.transpose());
- }
- }
-
- for (Matrix::Index x = 0; x < interp_uni.rows(); ++x) {
- // \sum_{contexts} B_I(context) \sum_x p_I(x) log p_i(x) log p_j(x)
- hessian.noalias() += sum_B_I * interp_uni(x) * ln_unigrams_.row(x).transpose() * ln_unigrams_.row(x);
- }
- return exp((neg_correct_summed_.dot(weights) + sum_ln_Z_context) / static_cast<double>(instances_.size()));
-}
-
-}} // namespaces
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/tune_derivatives.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/tune_derivatives.hh b/ext/kenlm/lm/interpolate/tune_derivatives.hh
deleted file mode 100644
index 40c058e..0000000
--- a/ext/kenlm/lm/interpolate/tune_derivatives.hh
+++ /dev/null
@@ -1,30 +0,0 @@
-#ifndef LM_INTERPOLATE_TUNE_DERIVATIVES_H
-#define LM_INTERPOLATE_TUNE_DERIVATIVES_H
-
-#include "lm/interpolate/tune_instance.hh"
-
-#include <Eigen/Core>
-#include <cmath>
-
-namespace lm { namespace interpolate {
-
-class ComputeDerivative {
- public:
- explicit ComputeDerivative(const util::FixedArray<Instance> &instances, const Matrix &ln_unigrams, WordIndex bos);
-
- Accum Iteration(const Vector &weights, Vector &gradient, Matrix &hessian);
-
- private:
- const util::FixedArray<Instance> &instances_;
- const Matrix &ln_unigrams_;
-
- const WordIndex bos_;
-
- // neg_correct_summed_(i) = -\sum_n ln p_i(w_n | w_1^{n-1})
- Vector neg_correct_summed_;
-};
-
-}} // namespaces
-
-#endif // LM_INTERPOLATE_TUNE_DERIVATIVES_H
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/tune_derivatives_test.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/tune_derivatives_test.cc b/ext/kenlm/lm/interpolate/tune_derivatives_test.cc
deleted file mode 100644
index 75c0d12..0000000
--- a/ext/kenlm/lm/interpolate/tune_derivatives_test.cc
+++ /dev/null
@@ -1,89 +0,0 @@
-#include "lm/interpolate/tune_derivatives.hh"
-
-#include "lm/interpolate/tune_instance.hh"
-
-#define BOOST_TEST_MODULE DerivativeTest
-#include <boost/test/unit_test.hpp>
-
-namespace lm { namespace interpolate { namespace {
-
-BOOST_AUTO_TEST_CASE(Small) {
- // Three vocabulary words plus <s>, two models.
- Matrix unigrams(4, 2);
- unigrams <<
- 0.1, 0.6,
- 0.4, 0.3,
- 0.5, 0.1,
- // <s>
- 1.0, 1.0;
- unigrams = unigrams.array().log();
-
- // One instance
- util::FixedArray<Instance> instances(1);
- instances.push_back(2);
- Instance &instance = instances.back();
-
- instance.ln_backoff << 0.2, 0.4;
- instance.ln_backoff = instance.ln_backoff.array().log();
-
- // Sparse cases: model 0 word 2 and model 1 word 1.
-
- // Assuming that model 1 only matches word 1, this is p_1(1 | context)
- Accum model_1_word_1 = 1.0 - .6 * .4 - .1 * .4;
-
- // We'll suppose correct has WordIndex 1, which backs off in model 0, and matches in model 1
- instance.ln_correct << (0.4 * 0.2), model_1_word_1;
- instance.ln_correct = instance.ln_correct.array().log();
-
- Accum model_0_word_2 = 1.0 - .1 * .2 - .4 * .2;
-
- instance.extension_words.push_back(1);
- instance.extension_words.push_back(2);
- instance.ln_extensions.resize(2, 2);
- instance.ln_extensions <<
- (0.4 * 0.2), model_1_word_1,
- model_0_word_2, 0.1 * 0.4;
- instance.ln_extensions = instance.ln_extensions.array().log();
-
- ComputeDerivative compute(instances, unigrams, 3);
- Vector weights(2);
- weights << 0.9, 1.2;
-
- Vector gradient(2);
- Matrix hessian(2,2);
- compute.Iteration(weights, gradient, hessian);
-
- // p_I(x | context)
- Vector p_I(3);
- p_I <<
- pow(0.1 * 0.2, 0.9) * pow(0.6 * 0.4, 1.2),
- pow(0.4 * 0.2, 0.9) * pow(model_1_word_1, 1.2),
- pow(model_0_word_2, 0.9) * pow(0.1 * 0.4, 1.2);
- p_I /= p_I.sum();
-
- Vector expected_gradient = -instance.ln_correct;
- expected_gradient(0) += p_I(0) * log(0.1 * 0.2);
- expected_gradient(0) += p_I(1) * log(0.4 * 0.2);
- expected_gradient(0) += p_I(2) * log(model_0_word_2);
- BOOST_CHECK_CLOSE(expected_gradient(0), gradient(0), 0.01);
-
- expected_gradient(1) += p_I(0) * log(0.6 * 0.4);
- expected_gradient(1) += p_I(1) * log(model_1_word_1);
- expected_gradient(1) += p_I(2) * log(0.1 * 0.4);
- BOOST_CHECK_CLOSE(expected_gradient(1), gradient(1), 0.01);
-
- Matrix expected_hessian(2, 2);
- expected_hessian(1, 0) =
- // First term
- p_I(0) * log(0.1 * 0.2) * log(0.6 * 0.4) +
- p_I(1) * log(0.4 * 0.2) * log(model_1_word_1) +
- p_I(2) * log(model_0_word_2) * log(0.1 * 0.4);
- expected_hessian(1, 0) -=
- (p_I(0) * log(0.1 * 0.2) + p_I(1) * log(0.4 * 0.2) + p_I(2) * log(model_0_word_2)) *
- (p_I(0) * log(0.6 * 0.4) + p_I(1) * log(model_1_word_1) + p_I(2) * log(0.1 * 0.4));
- expected_hessian(0, 1) = expected_hessian(1, 0);
- BOOST_CHECK_CLOSE(expected_hessian(1, 0), hessian(1, 0), 0.01);
- BOOST_CHECK_CLOSE(expected_hessian(0, 1), hessian(0, 1), 0.01);
-}
-
-}}} // namespaces
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/tune_instance.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/tune_instance.cc b/ext/kenlm/lm/interpolate/tune_instance.cc
deleted file mode 100644
index f1c9924..0000000
--- a/ext/kenlm/lm/interpolate/tune_instance.cc
+++ /dev/null
@@ -1,354 +0,0 @@
-#include "lm/interpolate/tune_instance.hh"
-
-#include "lm/common/model_buffer.hh"
-#include "lm/common/ngram_stream.hh"
-#include "lm/common/renumber.hh"
-#include "lm/enumerate_vocab.hh"
-#include "lm/interpolate/merge_vocab.hh"
-#include "lm/interpolate/universal_vocab.hh"
-#include "lm/lm_exception.hh"
-#include "util/file_piece.hh"
-#include "util/murmur_hash.hh"
-#include "util/stream/chain.hh"
-#include "util/tokenize_piece.hh"
-
-#include <boost/unordered_map.hpp>
-
-#include <cmath>
-#include <limits>
-#include <vector>
-
-namespace lm { namespace interpolate {
-
-// An extension without backoff weights applied yet.
-#pragma pack(push)
-#pragma pack(1)
-struct InitialExtension {
- Extension ext;
- // Order from which it came.
- uint8_t order;
-};
-#pragma pack(pop)
-
-// Intended use
-// For each model:
-// stream through orders jointly in suffix order:
-// Call MatchedBackoff for full matches.
-// Call Exit when the context matches.
-// Call FinishModel with the unigram probability of the correct word, get full
-// probability in return.
-// Use Backoffs to adjust records that were written to the stream.
-class InstanceMatch {
- public:
- InstanceMatch(ModelIndex models, uint8_t max_order, const WordIndex correct)
- : seen_(std::numeric_limits<WordIndex>::max()),
- backoffs_(Matrix::Zeros(models, max_order)),
- correct_(correct), correct_from_(1), correct_ln_prob_(std::numeric_limits<float>::quiet_NaN()) {}
-
- void MatchedBackoff(ModelIndex model, uint8_t order, float ln_backoff) {
- backoffs_(model, order - 1) = ln_backoff;
- }
-
- // We only want the highest-order matches, which are the first to be exited for a given word.
- void Exit(const InitialExtension &from, util::stream::Stream &out) {
- if (from.ext.word == seen_) return;
- seen_ = from.ext.word;
- *static_cast<InitialExtension*>(out.Get()) = from;
- ++out;
- if (UTIL_UNLIKELY(correct_ == from.ext.word)) {
- correct_from_ = from.order;
- correct_ln_prob_ = from.ext.ln_prob;
- }
- }
-
- WordIndex Correct() const { return correct_; }
-
- // Call this after each model has been passed through. The
- float FinishModel(ModelIndex model, float correct_ln_unigram) {
- seen_ = std::numeric_limits<WordIndex>::max();
- // Turn backoffs into multiplied values (added in log space).
- // So backoffs_(model, order - 1) is the penalty for matching order.
- float accum = 0.0;
- for (int order = backoffs_.cols() - 1; order >= 0; --order) {
- accum += backoffs_(model, order);
- backoffs_(model, order) = accum;
- }
- if (correct_from_ == 1) {
- correct_ln_prob_ = correct_ln_unigram;
- }
- if (correct_from_ - 1 < backoffs_.cols()) {
- correct_ln_prob_ += backoffs_(model, correct_from_ - 1);
- }
- correct_from_ = 1;
- return correct_ln_prob_;
- }
-
- const Matrix &Backoffs() const {
- return backoffs_;
- }
-
- private:
- // What's the last word we've seen? Used to act only on exiting the longest match.
- WordIndex seen_;
-
- Matrix backoffs_;
-
- const WordIndex correct_;
-
- // These only apply to the most recent model.
- uint8_t correct_from_;
-
- float correct_ln_prob_;
-};
-
-namespace {
-
-// Forward information to multiple instances of a context.
-class DispatchContext {
- public:
- void Register(InstanceMatch &context) {
- registered_.push_back(&context);
- }
-
- void MatchedBackoff(uint8_t order, float ln_backoff) {
- for (std::vector<InstanceMatch*>::iterator i = registered_.begin(); i != registered_.end(); ++i)
- (*i)->MatchedBackoff(order, ln_backoff);
- }
-
- void Exit(const InitialExtension &from, util::stream::Stream &out) {
- for (std::vector<InstanceMatch*>::iterator i = registered_.begin(); i != registered_.end(); ++i) {
- (*i)->Exit(from, out);
- }
- }
-
- private:
- std::vector<InstanceMatch*> registered_;
-};
-
-// Map from n-gram hash to contexts in the tuning data.
-typedef boost::unordered_map<uint64_t, DispatchContext> ContextMap;
-
-class ApplyBackoffs {
- public:
- explicit ApplyBackoffs(const InstanceMatch *backoffs) : backoffs_(backoffs) {}
-
- void Run(const util::stream::ChainPosition &position) {
- for (util::stream::Stream stream(position); stream; ++stream) {
- InitialExtension &ini = *reinterpret_cast<InitialExtension*>(stream.Get());
- ini.ext.ln_prob += backoffs_[ini.ext.instance]
- }
- }
-
- private:
- const InstanceMatch *backoffs_;
-};
-
-Instances::ReadExtensions(util::stream::Chain &on) {
- if (extensions_first_.get()) {
- // Lazy sort and save a sorted copy to disk. TODO: cut down on record size by stripping out order information?
- extensions_first_->Output(on);
- extensions_first_->reset();
- // TODO: apply backoff data!!!!
-
- extensions_subsequent_.reset(new util::stream::FileBuffer(util::MakeTemp(sorting_config_.temp_prefix)));
- on >> extensions_subsequent_->Sink();
- } else {
- on >> extensions_subsequent_->Source();
- }
-}
-
-class UnigramLoader {
- public:
- UnigramLoader(ContextMap &contexts_for_backoffs, Matrix &ln_probs, std::size_t model_number)
- : map_(contexts_for_backoffs),
- prob_(ln_probs.col(model_number)) {}
-
- void Run(const util::stream::ChainPosition &position) {
- // TODO handle the case of a unigram model?
- NGramStream<ProbBackoff> input(position);
- assert(input);
- Accum unk = input->Value().prob * M_LN10;
- WordIndex previous = 0;
- for (; input; ++input) {
- WordIndex word = *input->begin();
- prob_.segment(previous, word - previous) = Vector::Constant(word - previous, unk);
- prob_(word) = input->Value().prob * M_LN10;
- ContextMap::iterator i = map_.find(util::MurmurHashNative(input->begin(), sizeof(WordIndex)));
- if (i != map_.end()) {
- i->second.MatchedBackoff(1, input->Value().backoff * M_LN10);
- }
- previous = word + 1;
- }
- prob_.segment(previous, prob_.rows() - previous) = Vector::Constant(prob_.rows() - previous, unk);
- }
-
- private:
- ContextMap &map_;
- Matrix::ColXpr prob_;
- std::size_t model_;
-};
-
-class MiddleLoader {
- public:
- explicit MiddleLoader(ContextMap &map)
- : map_(map) {}
-
- void Run(const util::stream::ChainPosition &position) {
- NGramStream<ProbBackoff> input(position);
- const std::size_t full_size = (uint8_t*)input->end() - (uint8_t*)input->begin();
- const std::size_t context_size = full_size - sizeof(WordIndex);
- ContextMap::iterator i;
- for (; input; ++input) {
- i = map_.find(util::MurmurHashNative(input->begin(), full_size));
- if (i != map_.end()) {
- i->second.MatchedBackoff(input->Order(), input->Value().backoff * M_LN10);
- }
- i = map_.find(util::MurmurHashNative(input->begin(), context_size));
- if (i != map_.end()) {
- i->second.MatchedContext(input->Order(), *(input->end() - 1), input->Value().prob * M_LN10);
- }
- }
- }
-
- private:
- ContextMap &map_;
-};
-
-class HighestLoader {
- public:
- HighestLoader(ContextMap &map, uint8_t order)
- : map_(map), order_(order) {}
-
- void Run(const util::stream::ChainPosition &position) {
- ContextMap::iterator i;
- const std::size_t context_size = sizeof(WordIndex) * (order_ - 1);
- for (ProxyStream<NGram<float> > input(position, NGram<float>(NULL, order_)); input; ++input) {
- i = map_.find(util::MurmurHashNative(input->begin(), context_size));
- if (i != map_.end()) {
- i->second.MatchedContext(order_, *(input->end() - 1), input->Value() * M_LN10);
- }
- }
- }
-
- private:
- ContextMap &map_;
- const uint8_t order_;
-};
-
-class IdentifyTuning : public EnumerateVocab {
- public:
- IdentifyTuning(int tuning_file, std::vector<WordIndex> &out) : indices_(out) {
- indices_.clear();
- StringPiece line;
- std::size_t counter = 0;
- std::vector<std::size_t> &eos = words_[util::MurmurHashNative("</s>", 4)];
- for (util::FilePiece f(tuning_file); f.ReadLineOrEOF(line);) {
- for (util::TokenIter<util::BoolCharacter, true> word(line, util::kSpaces); word; ++word) {
- UTIL_THROW_IF(*word == "<s>" || *word == "</s>", FormatLoadException, "Illegal word in tuning data: " << *word);
- words_[util::MurmurHashNative(word->data(), word->size())].push_back(counter++);
- }
- eos.push_back(counter++);
- }
- // Also get <s>
- indices_.resize(counter + 1);
- words_[util::MurmurHashNative("<s>", 3)].push_back(indices_.size() - 1);
- }
-
- void Add(WordIndex id, const StringPiece &str) {
- boost::unordered_map<uint64_t, std::vector<std::size_t> >::iterator i = words_.find(util::MurmurHashNative(str.data(), str.size()));
- if (i != words_.end()) {
- for (std::vector<std::size_t>::iterator j = i->second.begin(); j != i->second.end(); ++j) {
- indices_[*j] = id;
- }
- }
- }
-
- WordIndex FinishGetBOS() {
- WordIndex ret = indices_.back();
- indices_.pop_back();
- return ret;
- }
-
- private:
- std::vector<WordIndex> &indices_;
-
- boost::unordered_map<uint64_t, std::vector<std::size_t> > words_;
-};
-
-} // namespace
-
-Instance::Instance(std::size_t num_models) : ln_backoff(num_models), ln_correct(num_models), ln_extensions(0, num_models) {}
-
-WordIndex LoadInstances(int tuning_file, const std::vector<StringPiece> &model_names, util::FixedArray<Instance> &instances, Matrix &ln_unigrams) {
- util::FixedArray<ModelBuffer> models(model_names.size());
- std::vector<WordIndex> vocab_sizes;
- vocab_sizes.reserve(model_names.size());
- util::FixedArray<util::scoped_fd> vocab_files(model_names.size());
- std::size_t max_order = 0;
- for (std::vector<StringPiece>::const_iterator i = model_names.begin(); i != model_names.end(); ++i) {
- models.push_back(*i);
- vocab_sizes.push_back(models.back().Counts()[0]);
- vocab_files.push_back(models.back().StealVocabFile());
- max_order = std::max(max_order, models.back().Order());
- }
- UniversalVocab vocab(vocab_sizes);
- std::vector<WordIndex> tuning_words;
- WordIndex bos;
- WordIndex combined_vocab_size;
- {
- IdentifyTuning identify(tuning_file, tuning_words);
- combined_vocab_size = MergeVocab(vocab_files, vocab, identify);
- bos = identify.FinishGetBOS();
- }
-
- instances.Init(tuning_words.size());
- util::FixedArray<InstanceBuilder> builders(tuning_words.size());
- std::vector<WordIndex> context;
- context.push_back(bos);
-
- // Populate the map from contexts to instance builders.
- ContextMap cmap;
- const WordIndex eos = tuning_words.back();
- for (std::size_t i = 0; i < tuning_words.size(); ++i) {
- instances.push_back(model_names.size());
- builders.push_back(tuning_words[i], max_order);
- for (std::size_t j = 0; j < context.size(); ++j) {
- cmap[util::MurmurHashNative(&context[j], sizeof(WordIndex) * (context.size() - j))].Register(builders.back());
- }
- // Prepare for next word.
- if (tuning_words[i] == eos) {
- context.clear();
- context.push_back(bos);
- } else {
- if (context.size() == max_order) {
- context.erase(context.begin());
- }
- context.push_back(tuning_words[i]);
- }
- }
-
- ln_unigrams.resize(combined_vocab_size, models.size());
-
- // Scan through input files. Sadly not parallel due to an underlying hash table.
- for (std::size_t m = 0; m < models.size(); ++m) {
- for (std::size_t order = 1; order <= models[m].Order(); ++order) {
- util::stream::Chain chain(util::stream::ChainConfig(sizeof(ProbBackoff) + order * sizeof(WordIndex), 2, 64 * 1048576));
- models[m].Source(order - 1, chain);
- chain >> Renumber(vocab.Mapping(m), order);
- if (order == 1) {
- chain >> UnigramLoader(cmap, ln_unigrams, m);
- } else if (order < models[m].Order()) {
- chain >> MiddleLoader(cmap);
- } else {
- chain >> HighestLoader(cmap, order);
- }
- }
- for (std::size_t instance = 0; instance < tuning_words.size(); ++instance) {
- builders[instance].Dump(m, ln_unigrams, instances[instance]);
- }
- ln_unigrams(bos, m) = -99; // Does not matter as long as it does not produce nans since tune_derivatives sets this to zero.
- }
- return bos;
-}
-
-}} // namespaces
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/tune_instance.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/tune_instance.hh b/ext/kenlm/lm/interpolate/tune_instance.hh
deleted file mode 100644
index c11eec9..0000000
--- a/ext/kenlm/lm/interpolate/tune_instance.hh
+++ /dev/null
@@ -1,86 +0,0 @@
-#ifndef LM_INTERPOLATE_TUNE_INSTANCE_H
-#define LM_INTERPOLATE_TUNE_INSTANCE_H
-
-#include "lm/interpolate/tune_matrix.hh"
-#include "lm/word_index.hh"
-#include "util/scoped.hh"
-#include "util/stream/config.hh"
-#include "util/string_piece.hh"
-
-#include <boost/optional.hpp>
-
-#include <vector>
-
-namespace util { namespace stream {
-template <class S, class T> class Sort;
-class Chain;
-class FileBuffer;
-}} // namespaces
-
-namespace lm { namespace interpolate {
-
-typedef uint32_t InstanceIndex;
-typedef uint32_t ModelIndex;
-
-struct Extension {
- // Which tuning instance does this belong to?
- InstanceIndex instance;
- WordIndex word;
- ModelIndex model;
- // ln p_{model} (word | context(instance))
- float ln_prob;
-
- bool operator<(const Extension &other) const {
- if (instance != other.instance)
- return instance < other.instance;
- if (word != other.word)
- return word < other.word;
- if (model != other.model)
- return model < other.model;
- return false;
- }
-};
-
-class Instances {
- public:
- Instances(int tune_file, const std::vector<StringPiece> &model_names);
-
- Eigen::ConstRowXpr Backoffs(InstanceIndex instance) const {
- return ln_backoffs_.row(instance);
- }
-
- const Vector &CorrectGradientTerm() const { return neg_ln_correct_sum_; }
-
- const Matrix &LNUnigrams() const { return ln_unigrams_; }
-
- void ReadExtensions(util::stream::Chain &to);
-
- private:
- // backoffs_(instance, model) is the backoff all the way to unigrams.
- typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor> BackoffMatrix;
- BackoffMatrix ln_backoffs_;
-
- // neg_correct_sum_(model) = -\sum_{instances} ln p_{model}(correct(instance) | context(instance)).
- // This appears as a term in the gradient.
- Vector neg_ln_correct_sum_;
-
- // unigrams_(word, model) = ln p_{model}(word).
- Matrix ln_unigrams_;
-
- struct ExtensionCompare {
- bool operator()(const void *f, const void *s) const {
- return reinterpret_cast<const Extension &>(f) < reinterpret_cast<const Extension &>(s);
- }
- };
-
- // This is the source of data for the first iteration.
- util::scoped_ptr<util::stream::Sort<ExtensionCompare> > extensions_first_;
-
- // Source of data for subsequent iterations. This contains already-sorted data.
- util::scoped_ptr<util::stream::FileBuffer> extensions_subsequent_;
-
- const util::stream::SortConfig sorting_config_;
-};
-
-}} // namespaces
-#endif // LM_INTERPOLATE_TUNE_INSTANCE_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/tune_instance_data/generate.sh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/tune_instance_data/generate.sh b/ext/kenlm/lm/interpolate/tune_instance_data/generate.sh
deleted file mode 100755
index d725572..0000000
--- a/ext/kenlm/lm/interpolate/tune_instance_data/generate.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/bash
-../../../bin/lmplz --discount_fallback -o 3 -S 100M --intermediate toy0 --arpa toy0.arpa <<EOF
-a a b a
-b a a b
-EOF
-../../../bin/lmplz --discount_fallback -o 3 -S 100M --intermediate toy1 --arpa toy1.arpa <<EOF
-a a b b b b b b b
-c
-EOF
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/tune_instance_data/toy0.1
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/tune_instance_data/toy0.1 b/ext/kenlm/lm/interpolate/tune_instance_data/toy0.1
deleted file mode 100644
index 1b66c51..0000000
Binary files a/ext/kenlm/lm/interpolate/tune_instance_data/toy0.1 and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/tune_instance_data/toy0.2
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/tune_instance_data/toy0.2 b/ext/kenlm/lm/interpolate/tune_instance_data/toy0.2
deleted file mode 100644
index d735b1c..0000000
Binary files a/ext/kenlm/lm/interpolate/tune_instance_data/toy0.2 and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/tune_instance_data/toy0.3
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/tune_instance_data/toy0.3 b/ext/kenlm/lm/interpolate/tune_instance_data/toy0.3
deleted file mode 100644
index 2d97aa3..0000000
Binary files a/ext/kenlm/lm/interpolate/tune_instance_data/toy0.3 and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/tune_instance_data/toy0.kenlm_intermediate
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/tune_instance_data/toy0.kenlm_intermediate b/ext/kenlm/lm/interpolate/tune_instance_data/toy0.kenlm_intermediate
deleted file mode 100644
index 8513475..0000000
--- a/ext/kenlm/lm/interpolate/tune_instance_data/toy0.kenlm_intermediate
+++ /dev/null
@@ -1,3 +0,0 @@
-KenLM intermediate binary file
-Counts 5 7 7
-Payload pb
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/tune_instance_data/toy0.vocab
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/tune_instance_data/toy0.vocab b/ext/kenlm/lm/interpolate/tune_instance_data/toy0.vocab
deleted file mode 100644
index 520c0f9..0000000
Binary files a/ext/kenlm/lm/interpolate/tune_instance_data/toy0.vocab and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/tune_instance_data/toy1.1
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/tune_instance_data/toy1.1 b/ext/kenlm/lm/interpolate/tune_instance_data/toy1.1
deleted file mode 100644
index a50cec6..0000000
Binary files a/ext/kenlm/lm/interpolate/tune_instance_data/toy1.1 and /dev/null differ
[34/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/stage.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/stage.jam b/ext/kenlm/jam-files/boost-build/tools/stage.jam
deleted file mode 100644
index 8d005ae..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/stage.jam
+++ /dev/null
@@ -1,519 +0,0 @@
-# Copyright 2003 Dave Abrahams
-# Copyright 2005, 2006 Rene Rivera
-# Copyright 2002, 2003, 2004, 2005, 2006 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# This module defines the 'install' rule, used to copy a set of targets to a
-# single location.
-
-import "class" : new ;
-import feature ;
-import generators ;
-import path ;
-import project ;
-import targets ;
-import type ;
-import types/register ;
-import virtual-target ;
-
-
-feature.feature <install-dependencies> : off on : incidental ;
-feature.feature <install-type> : : free incidental ;
-feature.feature <install-source-root> : : free path ;
-feature.feature <so-version> : : free incidental ;
-
-# If 'on', version symlinks for shared libraries will not be created. Affects
-# Unix builds only.
-feature.feature <install-no-version-symlinks> : on : optional incidental ;
-
-
-class install-target-class : basic-target
-{
- import "class" : new ;
- import feature ;
- import generators ;
- import path ;
- import project ;
- import property ;
- import property-set ;
- import stage ;
- import type ;
-
- rule __init__ ( name-and-dir : project : sources * : requirements * :
- default-build * : usage-requirements * )
- {
- # The usage-requirements specified here are ignored but are taken as a
- # parameter to have this metatarget class have the same standard
- # instantiation interface as all the other Boost Build metatarget
- # classes.
- basic-target.__init__ $(name-and-dir) : $(project) : $(sources) :
- $(requirements) : $(default-build) ;
- }
-
- # If <location> is not set, sets it based on the project data.
- #
- rule update-location ( property-set )
- {
- local loc = [ $(property-set).get <location> ] ;
- if ! $(loc)
- {
- loc = [ path.root $(self.name) [ $(self.project).get location ] ] ;
- property-set = [ $(property-set).add-raw $(loc:G=<location>) ] ;
- }
-
- return $(property-set) ;
- }
-
- # Takes a target that is installed and a property set which is used when
- # installing.
- #
- rule adjust-properties ( target : build-property-set )
- {
- local ps-raw ;
- local a = [ $(target).action ] ;
- if $(a)
- {
- local ps = [ $(a).properties ] ;
- ps-raw = [ $(ps).raw ] ;
-
- # Unless <hardcode-dll-paths>true is in properties, which can happen
- # only if the user has explicitly requested it, nuke all <dll-path>
- # properties.
- if [ $(build-property-set).get <hardcode-dll-paths> ] != true
- {
- ps-raw = [ property.change $(ps-raw) : <dll-path> ] ;
- }
-
- # If any <dll-path> properties were specified for installing, add
- # them.
- local l = [ $(build-property-set).get <dll-path> ] ;
- ps-raw += $(l:G=<dll-path>) ;
-
- # Also copy <linkflags> feature from current build set, to be used
- # for relinking.
- local l = [ $(build-property-set).get <linkflags> ] ;
- ps-raw += $(l:G=<linkflags>) ;
-
- # Remove the <tag> feature on original targets.
- ps-raw = [ property.change $(ps-raw) : <tag> ] ;
-
- # And <location>. If stage target has another stage target in
- # sources, then we shall get virtual targets with the <location>
- # property set.
- ps-raw = [ property.change $(ps-raw) : <location> ] ;
- }
-
- local d = [ $(build-property-set).get <dependency> ] ;
- ps-raw += $(d:G=<dependency>) ;
-
- local d = [ $(build-property-set).get <location> ] ;
- ps-raw += $(d:G=<location>) ;
-
- local ns = [ $(build-property-set).get <install-no-version-symlinks> ] ;
- ps-raw += $(ns:G=<install-no-version-symlinks>) ;
-
- local d = [ $(build-property-set).get <install-source-root> ] ;
- # Make the path absolute: we shall use it to compute relative paths and
- # making the path absolute will help.
- if $(d)
- {
- d = [ path.root $(d) [ path.pwd ] ] ;
- ps-raw += $(d:G=<install-source-root>) ;
- }
-
- if $(ps-raw)
- {
- return [ property-set.create $(ps-raw) ] ;
- }
- else
- {
- return [ property-set.empty ] ;
- }
- }
-
- rule construct ( name : source-targets * : property-set )
- {
- source-targets = [ targets-to-stage $(source-targets) :
- $(property-set) ] ;
-
- property-set = [ update-location $(property-set) ] ;
-
- local ename = [ $(property-set).get <name> ] ;
-
- if $(ename) && $(source-targets[2])
- {
- import errors : error : $(__name__) : errors.error ;
- errors.error When <name> property is used "in" 'install', only one
- source is allowed. ;
- }
-
- local result ;
- for local i in $(source-targets)
- {
- local staged-targets ;
-
- local new-properties = [ adjust-properties $(i) :
- $(property-set) ] ;
-
- # See if something special should be done when staging this type. It
- # is indicated by the presence of a special "INSTALLED_" type.
- local t = [ $(i).type ] ;
- if $(t) && [ type.registered INSTALLED_$(t) ]
- {
- if $(ename)
- {
- import errors : error : $(__name__) : errors.error ;
- errors.error In 'install': <name> property specified with
- target that requires relinking. ;
- }
- else
- {
- local targets = [ generators.construct $(self.project)
- $(name) : INSTALLED_$(t) : $(new-properties) : $(i) ] ;
- staged-targets += $(targets[2-]) ;
- }
- }
- else
- {
- staged-targets = [ stage.copy-file $(self.project) $(ename) :
- $(i) : $(new-properties) ] ;
- }
-
- if ! $(staged-targets)
- {
- import errors : error : $(__name__) : errors.error ;
- errors.error Unable to generate staged version of
- [ $(source).str ] ;
- }
-
- for t in $(staged-targets)
- {
- result += [ virtual-target.register $(t) ] ;
- }
- }
-
- return [ property-set.empty ] $(result) ;
- }
-
- # Given the list of source targets explicitly passed to 'stage', returns the
- # list of targets which must be staged.
- #
- rule targets-to-stage ( source-targets * : property-set )
- {
- local result ;
-
- # Traverse the dependencies, if needed.
- if [ $(property-set).get <install-dependencies> ] = "on"
- {
- source-targets = [ collect-targets $(source-targets) ] ;
- }
-
- # Filter the target types, if needed.
- local included-types = [ $(property-set).get <install-type> ] ;
- for local r in $(source-targets)
- {
- local ty = [ $(r).type ] ;
- if $(ty)
- {
- # Do not stage searched libs.
- if $(ty) != SEARCHED_LIB
- {
- if $(included-types)
- {
- if [ include-type $(ty) : $(included-types) ]
- {
- result += $(r) ;
- }
- }
- else
- {
- result += $(r) ;
- }
- }
- }
- else if ! $(included-types)
- {
- # Do not install typeless targets if there is an explicit list
- # of allowed types.
- result += $(r) ;
- }
- }
-
- return $(result) ;
- }
-
- # CONSIDER: figure out why we can not use virtual-target.traverse here.
- #
- rule collect-targets ( targets * )
- {
- # Find subvariants
- local s ;
- for local t in $(targets)
- {
- s += [ $(t).creating-subvariant ] ;
- }
- s = [ sequence.unique $(s) ] ;
-
- local result = [ new set ] ;
- $(result).add $(targets) ;
-
- for local i in $(s)
- {
- $(i).all-referenced-targets $(result) ;
- }
- local result2 ;
- for local r in [ $(result).list ]
- {
- if $(r:G) != <use>
- {
- result2 += $(r:G=) ;
- }
- }
- DELETE_MODULE $(result) ;
- return [ sequence.unique $(result2) ] ;
- }
-
- # Returns true iff 'type' is subtype of some element of 'types-to-include'.
- #
- local rule include-type ( type : types-to-include * )
- {
- local found ;
- while $(types-to-include) && ! $(found)
- {
- if [ type.is-subtype $(type) $(types-to-include[1]) ]
- {
- found = true ;
- }
- types-to-include = $(types-to-include[2-]) ;
- }
-
- return $(found) ;
- }
-}
-
-
-# Creates a copy of target 'source'. The 'properties' object should have a
-# <location> property which specifies where the target must be placed.
-#
-rule copy-file ( project name ? : source : properties )
-{
- name ?= [ $(source).name ] ;
- local relative ;
-
- local new-a = [ new non-scanning-action $(source) : common.copy :
- $(properties) ] ;
- local source-root = [ $(properties).get <install-source-root> ] ;
- if $(source-root)
- {
- # Get the real path of the target. We probably need to strip relative
- # path from the target name at construction.
- local path = [ $(source).path ] ;
- path = [ path.root $(name:D) $(path) ] ;
- # Make the path absolute. Otherwise, it would be hard to compute the
- # relative path. The 'source-root' is already absolute, see the
- # 'adjust-properties' method above.
- path = [ path.root $(path) [ path.pwd ] ] ;
-
- relative = [ path.relative-to $(source-root) $(path) ] ;
- }
-
- # Note: Using $(name:D=$(relative)) might be faster here, but then we would
- # need to explicitly check that relative is not ".", otherwise we might get
- # paths like '<prefix>/boost/.', try to create it and mkdir would obviously
- # fail.
- name = [ path.join $(relative) $(name:D=) ] ;
-
- return [ new file-target $(name) exact : [ $(source).type ] : $(project) :
- $(new-a) ] ;
-}
-
-
-rule symlink ( name : project : source : properties )
-{
- local a = [ new action $(source) : symlink.ln : $(properties) ] ;
- local t = [ new file-target $(name) exact : [ $(source).type ] : $(project)
- : $(a) ] ;
- return [ virtual-target.register $(t) ] ;
-}
-
-
-rule relink-file ( project : source : property-set )
-{
- local action = [ $(source).action ] ;
- local cloned-action = [ virtual-target.clone-action $(action) : $(project) :
- "" : $(property-set) ] ;
- return [ $(cloned-action).targets ] ;
-}
-
-
-# Declare installed version of the EXE type. Generator for this type will cause
-# relinking to the new location.
-type.register INSTALLED_EXE : : EXE ;
-
-
-class installed-exe-generator : generator
-{
- import type ;
- import property-set ;
- import modules ;
- import stage ;
-
- rule __init__ ( )
- {
- generator.__init__ install-exe : EXE : INSTALLED_EXE ;
- }
-
- rule run ( project name ? : property-set : source : multiple ? )
- {
- local stage-rule = stage.copy-file ;
-
- if ! [ $(property-set).get <os> ] in NT CYGWIN &&
- ! [ $(property-set).get <target-os> ] in windows cygwin
- {
- # If dll-path properties have been changed for the stage target,
- # relink instead of copying.
- local a = [ $(source).action ] ;
- local p = [ $(a).properties ] ;
- local original = [ $(p).get <dll-path> ] ;
- local current = [ $(property-set).get <dll-path> ] ;
-
- if $(current) != $(original)
- {
- stage-rule = stage.relink-file ;
- }
- }
-
- return [ $(stage-rule) $(project) : $(source) : $(property-set) ] ;
- }
-}
-
-
-generators.register [ new installed-exe-generator ] ;
-
-
-# Installing a shared link on Unix might cause a creation of versioned symbolic
-# links.
-type.register INSTALLED_SHARED_LIB : : SHARED_LIB ;
-
-
-class installed-shared-lib-generator : generator
-{
- import type ;
- import property-set ;
- import modules ;
- import stage ;
-
- rule __init__ ( )
- {
- generator.__init__ install-shared-lib : SHARED_LIB :
- INSTALLED_SHARED_LIB ;
- }
-
- rule run ( project name ? : property-set : source : multiple ? )
- {
- if [ $(property-set).get <os> ] in NT CYGWIN ||
- [ $(property-set).get <target-os> ] in windows cygwin
- {
- local copied = [ stage.copy-file $(project) : $(source) :
- $(property-set) ] ;
- return [ virtual-target.register $(copied) ] ;
- }
- else
- {
- local a = [ $(source).action ] ;
- local copied ;
- if ! $(a)
- {
- # Non-derived file, just copy.
- copied = [ stage.copy-file $(project) : $(source) :
- $(property-set) ] ;
- }
- else
- {
- local cp = [ $(a).properties ] ;
- local current-dll-path = [ $(cp).get <dll-path> ] ;
- local new-dll-path = [ $(property-set).get <dll-path> ] ;
-
- if $(current-dll-path) != $(new-dll-path)
- {
- # Rpath changed, need to relink.
- copied = [ stage.relink-file $(project) : $(source) :
- $(property-set) ] ;
- }
- else
- {
- copied = [ stage.copy-file $(project) : $(source) :
- $(property-set) ] ;
- }
- }
-
- copied = [ virtual-target.register $(copied) ] ;
-
- local result = $(copied) ;
- # If the name is in the form NNN.XXX.YYY.ZZZ, where all 'X', 'Y' and
- # 'Z' are numbers, we need to create NNN.XXX and NNN.XXX.YYY
- # symbolic links.
- local m = [ MATCH
- (.*)\\.([0123456789]+)\\.([0123456789]+)\\.([0123456789]+)$ :
- [ $(copied).name ] ] ;
- if $(m)
- {
- # Symlink without version at all is used to make
- # -lsome_library work.
- result += [ stage.symlink $(m[1]) : $(project) : $(copied) :
- $(property-set) ] ;
-
- # Symlinks of some libfoo.N and libfoo.N.M are used so that
- # library can found at runtime, if libfoo.N.M.X has soname of
- # libfoo.N. That happens when the library makes some binary
- # compatibility guarantees. If not, it is possible to skip those
- # symlinks.
- local suppress = [ $(property-set).get
- <install-no-version-symlinks> ] ;
-
- if $(suppress) != "on"
- {
- result += [ stage.symlink $(m[1]).$(m[2]) : $(project) :
- $(copied) : $(property-set) ] ;
- result += [ stage.symlink $(m[1]).$(m[2]).$(m[3]) :
- $(project) : $(copied) : $(property-set) ] ;
- }
- }
-
- return $(result) ;
- }
- }
-}
-
-generators.register [ new installed-shared-lib-generator ] ;
-
-
-# Main target rule for 'install'.
-#
-rule install ( name : sources * : requirements * : default-build * )
-{
- local project = [ project.current ] ;
-
- # Unless the user has explicitly asked us to hardcode dll paths, add
- # <hardcode-dll-paths>false in requirements, to override default value.
- if ! <hardcode-dll-paths>true in $(requirements)
- {
- requirements += <hardcode-dll-paths>false ;
- }
-
- if <tag> in $(requirements:G)
- {
- import errors ;
- errors.user-error The <tag> property is not allowed for the 'install'
- rule. ;
- }
-
- targets.create-metatarget install-target-class : $(project) : $(name) :
- $(sources) : $(requirements) : $(default-build) ;
-}
-
-
-IMPORT $(__name__) : install : : install ;
-IMPORT $(__name__) : install : : stage ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/stlport.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/stlport.jam b/ext/kenlm/jam-files/boost-build/tools/stlport.jam
deleted file mode 100644
index ed0947c..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/stlport.jam
+++ /dev/null
@@ -1,309 +0,0 @@
-# Copyright Gennadiy Rozental
-# Copyright 2006 Rene Rivera
-# Copyright 2003, 2004, 2006 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-# The STLPort is usable by means of 'stdlib' feature. When
-# stdlib=stlport is specified, default version of STLPort will be used,
-# while stdlib=stlport-4.5 will use specific version.
-# The subfeature value 'hostios' means to use host compiler's iostreams.
-#
-# The specific version of stlport is selected by features:
-# The <runtime-link> feature selects between static and shared library
-# The <runtime-debugging>on selects STLPort with debug symbols
-# and stl debugging.
-# There's no way to use STLPort with debug symbols but without
-# stl debugging.
-
-# TODO: must implement selection of different STLPort installations based
-# on used toolset.
-# Also, finish various flags:
-#
-# This is copied from V1 toolset, "+" means "implemented"
-#+flags $(CURR_TOOLSET) DEFINES <stlport-iostream>off : _STLP_NO_OWN_IOSTREAMS=1 _STLP_HAS_NO_NEW_IOSTREAMS=1 ;
-#+flags $(CURR_TOOLSET) DEFINES <stlport-extensions>off : _STLP_NO_EXTENSIONS=1 ;
-# flags $(CURR_TOOLSET) DEFINES <stlport-anachronisms>off : _STLP_NO_ANACHRONISMS=1 ;
-# flags $(CURR_TOOLSET) DEFINES <stlport-cstd-namespace>global : _STLP_VENDOR_GLOBAL_CSTD=1 ;
-# flags $(CURR_TOOLSET) DEFINES <exception-handling>off : _STLP_NO_EXCEPTIONS=1 ;
-# flags $(CURR_TOOLSET) DEFINES <stlport-debug-alloc>on : _STLP_DEBUG_ALLOC=1 ;
-#+flags $(CURR_TOOLSET) DEFINES <runtime-build>debug : _STLP_DEBUG=1 _STLP_DEBUG_UNINITIALIZED=1 ;
-#+flags $(CURR_TOOLSET) DEFINES <runtime-link>dynamic : _STLP_USE_DYNAMIC_LIB=1 ;
-
-
-import feature : feature subfeature ;
-import project ;
-import "class" : new ;
-import targets ;
-import property-set ;
-import common ;
-import type ;
-
-# Make this module into a project.
-project.initialize $(__name__) ;
-project stlport ;
-
-# The problem: how to request to use host compiler's iostreams?
-#
-# Solution 1: Global 'stlport-iostream' feature.
-# That's ugly. Subfeature make more sense for stlport-specific thing.
-# Solution 2: Use subfeature with two values, one of which ("use STLPort iostream")
-# is default.
-# The problem is that such subfeature will appear in target paths, and that's ugly
-# Solution 3: Use optional subfeature with only one value.
-
-feature.extend stdlib : stlport ;
-feature.compose <stdlib>stlport : <library>/stlport//stlport ;
-
-# STLport iostreams or native iostreams
-subfeature stdlib stlport : iostream : hostios : optional propagated ;
-
-# STLport extensions
-subfeature stdlib stlport : extensions : noext : optional propagated ;
-
-# STLport anachronisms -- NOT YET SUPPORTED
-# subfeature stdlib stlport : anachronisms : on off ;
-
-# STLport debug allocation -- NOT YET SUPPORTED
-#subfeature stdlib stlport : debug-alloc : off on ;
-
-# Declare a special target class to handle the creation of search-lib-target
-# instances for STLport. We need a special class, because otherwise we'll have
-# - declare prebuilt targets for all possible toolsets. And by the time 'init'
-# is called we don't even know the list of toolsets that are registered
-# - when host iostreams are used, we really should produce nothing. It would
-# be hard/impossible to achieve this using prebuilt targets.
-
-class stlport-target-class : basic-target
-{
- import feature project type errors generators ;
- import set : difference ;
-
- rule __init__ ( project : headers ? : libraries * : version ? )
- {
- basic-target.__init__ stlport : $(project) ;
- self.headers = $(headers) ;
- self.libraries = $(libraries) ;
- self.version = $(version) ;
- self.version.5 = [ MATCH "^(5[.][0123456789]+).*" : $(version) ] ;
-
- local requirements ;
- requirements += <stdlib-stlport:version>$(self.version) ;
- self.requirements = [ property-set.create $(requirements) ] ;
- }
-
- rule generate ( property-set )
- {
- # Since this target is built with <stdlib>stlport, it will also
- # have <library>/stlport//stlport in requirements, which will
- # cause a loop in main target references. Remove that property
- # manually.
-
- property-set = [ property-set.create
- [ difference
- [ $(property-set).raw ] :
- <library>/stlport//stlport
- <stdlib>stlport
- ]
- ] ;
- return [ basic-target.generate $(property-set) ] ;
- }
-
- rule construct ( name : source-targets * : property-set )
- {
- # Deduce the name of stlport library, based on toolset and
- # debug setting.
- local raw = [ $(property-set).raw ] ;
- local hostios = [ feature.get-values <stdlib-stlport:iostream> : $(raw) ] ;
- local toolset = [ feature.get-values <toolset> : $(raw) ] ;
-
- if $(self.version.5)
- {
- # Version 5.x
-
- # STLport host IO streams no longer supported. So we always
- # need libraries.
-
- # name: stlport(stl)?[dg]?(_static)?.M.R
- local name = stlport ;
- if [ feature.get-values <runtime-debugging> : $(raw) ] = "on"
- {
- name += stl ;
- switch $(toolset)
- {
- case gcc* : name += g ;
- case darwin* : name += g ;
- case * : name += d ;
- }
- }
-
- if [ feature.get-values <runtime-link> : $(raw) ] = "static"
- {
- name += _static ;
- }
-
- # Starting with version 5.2.0, the STLport static libraries no
- # longer include a version number in their name
- local version.pre.5.2 = [ MATCH "^(5[.][01]+).*" : $(version) ] ;
- if $(version.pre.5.2) || [ feature.get-values <runtime-link> :
- $(raw) ] != "static"
- {
- name += .$(self.version.5) ;
- }
-
- name = $(name:J=) ;
-
- if [ feature.get-values <install-dependencies> : $(raw) ] = "on"
- {
- #~ Allow explicitly asking to install the STLport lib by
- #~ referring to it directly:
- #~ /stlport//stlport/<install-dependencies>on
- #~ This allows for install packaging of all libs one might need
- #~ for a standalone distribution.
- import path : make : path-make ;
- local runtime-link
- = [ feature.get-values <runtime-link> : $(raw) ] ;
- local lib-file.props
- = [ property-set.create $(raw) <link>$(runtime-link) ] ;
- local lib-file.prefix
- = [ type.generated-target-prefix $(runtime-link:U)_LIB :
- $(lib-file.props) ] ;
- local lib-file.suffix
- = [ type.generated-target-suffix $(runtime-link:U)_LIB :
- $(lib-file.props) ] ;
- lib-file.prefix
- ?= "" "lib" ;
- lib-file.suffix
- ?= "" ;
- local lib-file
- = [ GLOB $(self.libraries) [ modules.peek : PATH ] :
- $(lib-file.prefix)$(name).$(lib-file.suffix) ] ;
- lib-file
- = [ new file-reference [ path-make $(lib-file[1]) ] :
- $(self.project) ] ;
- lib-file
- = [ $(lib-file).generate "" ] ;
- local lib-file.requirements
- = [ targets.main-target-requirements
- [ $(lib-file.props).raw ] <file>$(lib-file[-1])
- : $(self.project) ] ;
- return [ generators.construct $(self.project) $(name) : LIB :
- $(lib-file.requirements) ] ;
- }
- else
- {
- #~ Otherwise, it is just regular library usage.
- return [ generators.construct
- $(self.project) $(name) : SEARCHED_LIB : $(property-set) ] ;
- }
- }
- else if ! $(hostios) && $(toolset) != msvc
- {
- # We don't need libraries if host istreams are used. For
- # msvc, automatic library selection will be used.
-
- # name: stlport_<toolset>(_stldebug)?
- local name = stlport ;
- name = $(name)_$(toolset) ;
- if [ feature.get-values <runtime-debugging> : $(raw) ] = "on"
- {
- name = $(name)_stldebug ;
- }
-
- return [ generators.construct
- $(self.project) $(name) : SEARCHED_LIB : $(property-set) ] ;
- }
- else
- {
- return [ property-set.empty ] ;
- }
- }
-
- rule compute-usage-requirements ( subvariant )
- {
- local usage-requirements =
- <include>$(self.headers)
- <dll-path>$(self.libraries)
- <library-path>$(self.libraries)
- ;
-
- local rproperties = [ $(subvariant).build-properties ] ;
- # CONSIDER: should this "if" sequence be replaced with
- # some use of 'property-map' class?
- if [ $(rproperties).get <runtime-debugging> ] = "on"
- {
- usage-requirements +=
- <define>_STLP_DEBUG=1
- <define>_STLP_DEBUG_UNINITIALIZED=1 ;
- }
- if [ $(rproperties).get <runtime-link> ] = "shared"
- {
- usage-requirements +=
- <define>_STLP_USE_DYNAMIC_LIB=1 ;
- }
- if [ $(rproperties).get <stdlib-stlport:extensions> ] = noext
- {
- usage-requirements +=
- <define>_STLP_NO_EXTENSIONS=1 ;
- }
- if [ $(rproperties).get <stdlib-stlport:iostream> ] = hostios
- {
- usage-requirements +=
- <define>_STLP_NO_OWN_IOSTREAMS=1
- <define>_STLP_HAS_NO_NEW_IOSTREAMS=1 ;
- }
- if $(self.version.5)
- {
- # Version 5.x
- if [ $(rproperties).get <threading> ] = "single"
- {
- # Since STLport5 doesn't normally support single-thread
- # we force STLport5 into the multi-thread mode. Hence
- # getting what other libs provide of single-thread code
- # linking against a multi-thread lib.
- usage-requirements +=
- <define>_STLP_THREADS=1 ;
- }
- }
-
- return [ property-set.create $(usage-requirements) ] ;
- }
-}
-
-rule stlport-target ( headers ? : libraries * : version ? )
-{
- local project = [ project.current ] ;
-
- targets.main-target-alternative
- [ new stlport-target-class $(project) : $(headers) : $(libraries)
- : $(version)
- ] ;
-}
-
-local .version-subfeature-defined ;
-
-# Initialize stlport support.
-rule init (
- version ? :
- headers : # Location of header files
- libraries * # Location of libraries, lib and bin subdirs of STLport.
- )
-{
- # FIXME: need to use common.check-init-parameters here.
- # At the moment, that rule always tries to define subfeature
- # of the 'toolset' feature, while we need to define subfeature
- # of <stdlib>stlport, so tweaks to check-init-parameters are needed.
- if $(version)
- {
- if ! $(.version-subfeature-defined)
- {
- feature.subfeature stdlib stlport : version : : propagated ;
- .version-subfeature-defined = true ;
- }
- feature.extend-subfeature stdlib stlport : version : $(version) ;
- }
-
- # Declare the main target for this STLPort version.
- stlport-target $(headers) : $(libraries) : $(version) ;
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/sun.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/sun.jam b/ext/kenlm/jam-files/boost-build/tools/sun.jam
deleted file mode 100644
index 0ca927d..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/sun.jam
+++ /dev/null
@@ -1,142 +0,0 @@
-# Copyright (C) Christopher Currie 2003. Permission to copy, use,
-# modify, sell and distribute this software is granted provided this
-# copyright notice appears in all copies. This software is provided
-# "as is" without express or implied warranty, and with no claim as
-# to its suitability for any purpose.
-
-import property ;
-import generators ;
-import os ;
-import toolset : flags ;
-import feature ;
-import type ;
-import common ;
-
-feature.extend toolset : sun ;
-toolset.inherit sun : unix ;
-generators.override sun.prebuilt : builtin.lib-generator ;
-generators.override sun.prebuilt : builtin.prebuilt ;
-generators.override sun.searched-lib-generator : searched-lib-generator ;
-
-feature.extend stdlib : sun-stlport ;
-feature.compose <stdlib>sun-stlport
- : <cxxflags>-library=stlport4 <linkflags>-library=stlport4
- ;
-
-rule init ( version ? : command * : options * )
-{
- local condition = [
- common.check-init-parameters sun : version $(version) ] ;
-
- command = [ common.get-invocation-command sun : CC
- : $(command) : "/opt/SUNWspro/bin" ] ;
-
- # Even if the real compiler is not found, put CC to
- # command line so that user see command line that would have being executed.
- command ?= CC ;
-
- common.handle-options sun : $(condition) : $(command) : $(options) ;
-
- command_c = $(command[1--2]) $(command[-1]:B=cc) ;
-
- toolset.flags sun CONFIG_C_COMMAND $(condition) : $(command_c) ;
-}
-
-# Declare generators
-generators.register-c-compiler sun.compile.c : C : OBJ : <toolset>sun ;
-generators.register-c-compiler sun.compile.c++ : CPP : OBJ : <toolset>sun ;
-
-# Declare flags and actions for compilation
-flags sun.compile OPTIONS <debug-symbols>on : -g ;
-flags sun.compile OPTIONS <profiling>on : -xprofile=tcov ;
-flags sun.compile OPTIONS <optimization>speed : -xO4 ;
-flags sun.compile OPTIONS <optimization>space : -xO2 -xspace ;
-flags sun.compile OPTIONS <threading>multi : -mt ;
-flags sun.compile OPTIONS <warnings>off : -erroff ;
-flags sun.compile OPTIONS <warnings>on : -erroff=%none ;
-flags sun.compile OPTIONS <warnings>all : -erroff=%none ;
-flags sun.compile OPTIONS <warnings-as-errors>on : -errwarn ;
-
-flags sun.compile.c++ OPTIONS <inlining>off : +d ;
-
-# The -m32 and -m64 options are supported starting
-# with Sun Studio 12. On earlier compilers, the
-# 'address-model' feature is not supported and should not
-# be used. Instead, use -xarch=generic64 command line
-# option.
-# See http://svn.boost.org/trac/boost/ticket/1186
-# for details.
-flags sun OPTIONS <address-model>32 : -m32 ;
-flags sun OPTIONS <address-model>64 : -m64 ;
-# On sparc, there's a difference between -Kpic
-# and -KPIC. The first is slightly more efficient,
-# but has the limits on the size of GOT table.
-# For minimal fuss on user side, we use -KPIC here.
-# See http://svn.boost.org/trac/boost/ticket/1186#comment:6
-# for detailed explanation.
-flags sun OPTIONS <link>shared : -KPIC ;
-
-flags sun.compile OPTIONS <cflags> ;
-flags sun.compile.c++ OPTIONS <cxxflags> ;
-flags sun.compile DEFINES <define> ;
-flags sun.compile INCLUDES <include> ;
-
-actions compile.c
-{
- "$(CONFIG_C_COMMAND)" $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
-}
-
-actions compile.c++
-{
- "$(CONFIG_COMMAND)" $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
-}
-
-# Declare flags and actions for linking
-flags sun.link OPTIONS <debug-symbols>on : -g ;
-# Strip the binary when no debugging is needed
-flags sun.link OPTIONS <debug-symbols>off : -s ;
-flags sun.link OPTIONS <profiling>on : -xprofile=tcov ;
-flags sun.link OPTIONS <threading>multi : -mt ;
-flags sun.link OPTIONS <linkflags> ;
-flags sun.link LINKPATH <library-path> ;
-flags sun.link FINDLIBS-ST <find-static-library> ;
-flags sun.link FINDLIBS-SA <find-shared-library> ;
-flags sun.link LIBRARIES <library-file> ;
-flags sun.link LINK-RUNTIME <runtime-link>static : static ;
-flags sun.link LINK-RUNTIME <runtime-link>shared : dynamic ;
-flags sun.link RPATH <dll-path> ;
-# On gcc, there are separate options for dll path at runtime and
-# link time. On Solaris, there's only one: -R, so we have to use
-# it, even though it's bad idea.
-flags sun.link RPATH <xdll-path> ;
-
-# The POSIX real-time library is always needed (nanosleep, clock_gettime etc.)
-flags sun.link FINDLIBS-SA : rt ;
-
-rule link ( targets * : sources * : properties * )
-{
- SPACE on $(targets) = " " ;
-}
-
-actions link bind LIBRARIES
-{
- "$(CONFIG_COMMAND)" $(OPTIONS) -L"$(LINKPATH)" -R"$(RPATH)" -o "$(<)" "$(>)" "$(LIBRARIES)" -Bdynamic -l$(FINDLIBS-SA) -Bstatic -l$(FINDLIBS-ST) -B$(LINK-RUNTIME)
-}
-
-# Slight mods for dlls
-rule link.dll ( targets * : sources * : properties * )
-{
- SPACE on $(targets) = " " ;
-}
-
-actions link.dll bind LIBRARIES
-{
- "$(CONFIG_COMMAND)" $(OPTIONS) -L"$(LINKPATH)" -R"$(RPATH)" -o "$(<)" -h$(<[1]:D=) -G "$(>)" "$(LIBRARIES)" -Bdynamic -l$(FINDLIBS-SA) -Bstatic -l$(FINDLIBS-ST) -B$(LINK-RUNTIME)
-}
-
-# Declare action for creating static libraries
-actions piecemeal archive
-{
- "$(CONFIG_COMMAND)" -xar -o "$(<)" "$(>)"
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/symlink.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/symlink.jam b/ext/kenlm/jam-files/boost-build/tools/symlink.jam
deleted file mode 100644
index b33e826..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/symlink.jam
+++ /dev/null
@@ -1,140 +0,0 @@
-# Copyright 2003 Dave Abrahams
-# Copyright 2002, 2003 Rene Rivera
-# Copyright 2002, 2003, 2004, 2005 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-# Defines the "symlink" special target. 'symlink' targets make symbolic links
-# to the sources.
-
-import targets modules path class os feature project property-set ;
-
-.count = 0 ;
-
-feature.feature symlink-location : project-relative build-relative : incidental ;
-
-# The class representing "symlink" targets.
-#
-class symlink-targets : basic-target
-{
- import numbers modules class property project path ;
-
- rule __init__ (
- project
- : targets *
- : sources *
- )
- {
- # Generate a fake name for now. Need unnamed targets eventually.
- local c = [ modules.peek symlink : .count ] ;
- modules.poke symlink : .count : [ numbers.increment $(c) ] ;
- local fake-name = symlink#$(c) ;
-
- basic-target.__init__ $(fake-name) : $(project) : $(sources) ;
-
- # Remember the targets to map the sources onto. Pad or truncate
- # to fit the sources given.
- self.targets = ;
- for local source in $(sources)
- {
- if $(targets)
- {
- self.targets += $(targets[1]) ;
- targets = $(targets[2-]) ;
- }
- else
- {
- self.targets += $(source) ;
- }
- }
-
- # The virtual targets corresponding to the given targets.
- self.virtual-targets = ;
- }
-
- rule construct ( name : source-targets * : property-set )
- {
- local i = 1 ;
- for local t in $(source-targets)
- {
- local s = $(self.targets[$(i)]) ;
- local a = [ class.new action $(t) : symlink.ln : $(property-set) ] ;
- local vt = [ class.new file-target $(s:D=)
- : [ $(t).type ] : $(self.project) : $(a) ] ;
-
- # Place the symlink in the directory relative to the project
- # location, instead of placing it in the build directory.
- if [ property.select <symlink-location> : [ $(property-set).raw ] ] = <symlink-location>project-relative
- {
- $(vt).set-path [ path.root $(s:D) [ $(self.project).get location ] ] ;
- }
-
- self.virtual-targets += $(vt) ;
- i = [ numbers.increment $(i) ] ;
- }
- return [ property-set.empty ] $(self.virtual-targets) ;
- }
-}
-
-# Creates a symbolic link from a set of targets to a set of sources.
-# The targets and sources map one to one. The symlinks generated are
-# limited to be the ones given as the sources. That is, the targets
-# are either padded or trimmed to equate to the sources. The padding
-# is done with the name of the corresponding source. For example::
-#
-# symlink : one two ;
-#
-# Is equal to::
-#
-# symlink one two : one two ;
-#
-# Names for symlink are relative to the project location. They cannot
-# include ".." path components.
-rule symlink (
- targets *
- : sources *
- )
-{
- local project = [ project.current ] ;
-
- return [ targets.main-target-alternative
- [ class.new symlink-targets $(project) : $(targets) :
- # Note: inline targets are not supported for symlink, intentionally,
- # since it's used to linking existing non-local targets.
- $(sources) ] ] ;
-}
-
-rule ln
-{
- local os ;
- if [ modules.peek : UNIX ] { os = UNIX ; }
- else { os ?= [ os.name ] ; }
- # Remember the path to make the link relative to where the symlink is located.
- local path-to-source = [ path.relative-to
- [ path.make [ on $(<) return $(LOCATE) ] ]
- [ path.make [ on $(>) return $(LOCATE) ] ] ] ;
- if $(path-to-source) = .
- {
- PATH_TO_SOURCE on $(<) = "" ;
- }
- else
- {
- PATH_TO_SOURCE on $(<) = [ path.native $(path-to-source) ] ;
- }
- ln-$(os) $(<) : $(>) ;
-}
-
-actions ln-UNIX
-{
- ln -f -s '$(>:D=:R=$(PATH_TO_SOURCE))' '$(<)'
-}
-
-# there is a way to do this; we fall back to a copy for now
-actions ln-NT
-{
- echo "NT symlinks not supported yet, making copy"
- del /f /q "$(<)" 2>nul >nul
- copy "$(>)" "$(<)" $(NULL_OUT)
-}
-
-IMPORT $(__name__) : symlink : : symlink ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/testing-aux.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/testing-aux.jam b/ext/kenlm/jam-files/boost-build/tools/testing-aux.jam
deleted file mode 100644
index 64ba003..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/testing-aux.jam
+++ /dev/null
@@ -1,220 +0,0 @@
-# This module is imported by testing.py. The definitions here are
-# too tricky to do in Python
-
-# Causes the 'target' to exist after bjam invocation if and only if all the
-# dependencies were successfully built.
-#
-rule expect-success ( target : dependency + : requirements * )
-{
- **passed** $(target) : $(sources) ;
-}
-IMPORT testing : expect-success : : testing.expect-success ;
-
-# Causes the 'target' to exist after bjam invocation if and only if all some of
-# the dependencies were not successfully built.
-#
-rule expect-failure ( target : dependency + : properties * )
-{
- local grist = [ MATCH ^<(.*)> : $(dependency:G) ] ;
- local marker = $(dependency:G=$(grist)*fail) ;
- (failed-as-expected) $(marker) ;
- FAIL_EXPECTED $(dependency) ;
- LOCATE on $(marker) = [ on $(dependency) return $(LOCATE) ] ;
- RMOLD $(marker) ;
- DEPENDS $(marker) : $(dependency) ;
- DEPENDS $(target) : $(marker) ;
- **passed** $(target) : $(marker) ;
-}
-IMPORT testing : expect-failure : : testing.expect-failure ;
-
-# The rule/action combination used to report successful passing of a test.
-#
-rule **passed**
-{
- # Force deletion of the target, in case any dependencies failed to build.
- RMOLD $(<) ;
-}
-
-
-# Used to create test files signifying passed tests.
-#
-actions **passed**
-{
- echo passed > "$(<)"
-}
-
-
-# Used to create replacement object files that do not get created during tests
-# that are expected to fail.
-#
-actions (failed-as-expected)
-{
- echo failed as expected > "$(<)"
-}
-
-# Runs executable 'sources' and stores stdout in file 'target'. Unless
-# --preserve-test-targets command line option has been specified, removes the
-# executable. The 'target-to-remove' parameter controls what should be removed:
-# - if 'none', does not remove anything, ever
-# - if empty, removes 'source'
-# - if non-empty and not 'none', contains a list of sources to remove.
-#
-rule capture-output ( target : source : properties * : targets-to-remove * )
-{
- output-file on $(target) = $(target:S=.output) ;
- LOCATE on $(target:S=.output) = [ on $(target) return $(LOCATE) ] ;
-
- # The INCLUDES kill a warning about independent target...
- INCLUDES $(target) : $(target:S=.output) ;
- # but it also puts .output into dependency graph, so we must tell jam it is
- # OK if it cannot find the target or updating rule.
- NOCARE $(target:S=.output) ;
-
- # This has two-fold effect. First it adds input files to the dependendency
- # graph, preventing a warning. Second, it causes input files to be bound
- # before target is created. Therefore, they are bound using SEARCH setting
- # on them and not LOCATE setting of $(target), as in other case (due to jam
- # bug).
- DEPENDS $(target) : [ on $(target) return $(INPUT_FILES) ] ;
-
- if $(targets-to-remove) = none
- {
- targets-to-remove = ;
- }
- else if ! $(targets-to-remove)
- {
- targets-to-remove = $(source) ;
- }
-
- if [ on $(target) return $(REMOVE_TEST_TARGETS) ]
- {
- TEMPORARY $(targets-to-remove) ;
- # Set a second action on target that will be executed after capture
- # output action. The 'RmTemps' rule has the 'ignore' modifier so it is
- # always considered succeeded. This is needed for 'run-fail' test. For
- # that test the target will be marked with FAIL_EXPECTED, and without
- # 'ignore' successful execution will be negated and be reported as
- # failure. With 'ignore' we do not detect a case where removing files
- # fails, but it is not likely to happen.
- RmTemps $(target) : $(targets-to-remove) ;
- }
-}
-
-
-if [ os.name ] = NT
-{
- .STATUS = %status% ;
- .SET_STATUS = "set status=%ERRORLEVEL%" ;
- .RUN_OUTPUT_NL = "echo." ;
- .STATUS_0 = "%status% EQU 0 (" ;
- .STATUS_NOT_0 = "%status% NEQ 0 (" ;
- .VERBOSE = "%verbose% EQU 1 (" ;
- .ENDIF = ")" ;
- .SHELL_SET = "set " ;
- .CATENATE = type ;
- .CP = copy ;
-}
-else
-{
- .STATUS = "$status" ;
- .SET_STATUS = "status=$?" ;
- .RUN_OUTPUT_NL = "echo" ;
- .STATUS_0 = "test $status -eq 0 ; then" ;
- .STATUS_NOT_0 = "test $status -ne 0 ; then" ;
- .VERBOSE = "test $verbose -eq 1 ; then" ;
- .ENDIF = "fi" ;
- .SHELL_SET = "" ;
- .CATENATE = cat ;
- .CP = cp ;
-}
-
-
-.VERBOSE_TEST = 0 ;
-if --verbose-test in [ modules.peek : ARGV ]
-{
- .VERBOSE_TEST = 1 ;
-}
-
-
-.RM = [ common.rm-command ] ;
-
-
-actions capture-output bind INPUT_FILES output-file
-{
- $(PATH_SETUP)
- $(LAUNCHER) "$(>)" $(ARGS) "$(INPUT_FILES)" > "$(output-file)" 2>&1
- $(.SET_STATUS)
- $(.RUN_OUTPUT_NL) >> "$(output-file)"
- echo EXIT STATUS: $(.STATUS) >> "$(output-file)"
- if $(.STATUS_0)
- $(.CP) "$(output-file)" "$(<)"
- $(.ENDIF)
- $(.SHELL_SET)verbose=$(.VERBOSE_TEST)
- if $(.STATUS_NOT_0)
- $(.SHELL_SET)verbose=1
- $(.ENDIF)
- if $(.VERBOSE)
- echo ====== BEGIN OUTPUT ======
- $(.CATENATE) "$(output-file)"
- echo ====== END OUTPUT ======
- $(.ENDIF)
- exit $(.STATUS)
-}
-
-IMPORT testing : capture-output : : testing.capture-output ;
-
-
-actions quietly updated ignore piecemeal together RmTemps
-{
- $(.RM) "$(>)"
-}
-
-
-.MAKE_FILE = [ common.file-creation-command ] ;
-
-actions unit-test
-{
- $(PATH_SETUP)
- $(LAUNCHER) "$(>)" $(ARGS) && $(.MAKE_FILE) "$(<)"
-}
-
-# Note that this rule may be called multiple times for a single target in case
-# there are multiple actions operating on the same target in sequence. One such
-# example are msvc exe targets first created by a linker action and then updated
-# with an embedded manifest file by a separate action.
-rule record-time ( target : source : start end user system )
-{
- local src-string = [$(source:G=:J=",")"] " ;
- USER_TIME on $(target) += $(src-string)$(user) ;
- SYSTEM_TIME on $(target) += $(src-string)$(system) ;
-
- # We need the following variables because attempting to perform such
- # variable expansion in actions would not work due to quotes getting treated
- # as regular characters.
- USER_TIME_SECONDS on $(target) += $(src-string)$(user)" seconds" ;
- SYSTEM_TIME_SECONDS on $(target) += $(src-string)$(system)" seconds" ;
-}
-
-# Calling this rule requests that Boost Build time how long it takes to build
-# the 'source' target and display the results both on the standard output and in
-# the 'target' file.
-#
-rule time ( target : sources + : properties * )
-{
- # Set up rule for recording timing information.
- __TIMING_RULE__ on $(sources) = testing.record-time $(target) ;
-
- # Make sure the sources get rebuilt any time we need to retrieve that
- # information.
- REBUILDS $(target) : $(sources) ;
-}
-
-
-actions time
-{
- echo user: $(USER_TIME)
- echo system: $(SYSTEM_TIME)
-
- echo user: $(USER_TIME_SECONDS) > "$(<)"
- echo system: $(SYSTEM_TIME_SECONDS) >> "$(<)"
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/testing.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/testing.jam b/ext/kenlm/jam-files/boost-build/tools/testing.jam
deleted file mode 100644
index 21e1bd1..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/testing.jam
+++ /dev/null
@@ -1,594 +0,0 @@
-# Copyright 2005 Dave Abrahams
-# Copyright 2002, 2003, 2004, 2005, 2006 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-# This module implements regression testing framework. It declares a number of
-# main target rules which perform some action and, if the results are OK,
-# creates an output file.
-#
-# The exact list of rules is:
-# 'compile' -- creates .test file if compilation of sources was
-# successful.
-# 'compile-fail' -- creates .test file if compilation of sources failed.
-# 'run' -- creates .test file is running of executable produced from
-# sources was successful. Also leaves behind .output file
-# with the output from program run.
-# 'run-fail' -- same as above, but .test file is created if running fails.
-#
-# In all cases, presence of .test file is an indication that the test passed.
-# For more convenient reporting, you might want to use C++ Boost regression
-# testing utilities (see http://www.boost.org/more/regression.html).
-#
-# For historical reason, a 'unit-test' rule is available which has the same
-# syntax as 'exe' and behaves just like 'run'.
-
-# Things to do:
-# - Teach compiler_status handle Jamfile.v2.
-# Notes:
-# - <no-warn> is not implemented, since it is Como-specific, and it is not
-# clear how to implement it
-# - std::locale-support is not implemented (it is used in one test).
-
-
-import alias ;
-import "class" ;
-import common ;
-import errors ;
-import feature ;
-import generators ;
-import os ;
-import path ;
-import project ;
-import property ;
-import property-set ;
-import regex ;
-import sequence ;
-import targets ;
-import toolset ;
-import type ;
-import virtual-target ;
-
-
-rule init ( )
-{
-}
-
-
-# Feature controling the command used to lanch test programs.
-feature.feature testing.launcher : : free optional ;
-
-feature.feature test-info : : free incidental ;
-feature.feature testing.arg : : free incidental ;
-feature.feature testing.input-file : : free dependency ;
-
-feature.feature preserve-test-targets : on off : incidental propagated ;
-
-# Register target types.
-type.register TEST : test ;
-type.register COMPILE : : TEST ;
-type.register COMPILE_FAIL : : TEST ;
-type.register RUN_OUTPUT : run ;
-type.register RUN : : TEST ;
-type.register RUN_FAIL : : TEST ;
-type.register LINK_FAIL : : TEST ;
-type.register LINK : : TEST ;
-type.register UNIT_TEST : passed : TEST ;
-
-
-# Declare the rules which create main targets. While the 'type' module already
-# creates rules with the same names for us, we need extra convenience: default
-# name of main target, so write our own versions.
-
-# Helper rule. Create a test target, using basename of first source if no target
-# name is explicitly passed. Remembers the created target in a global variable.
-#
-rule make-test ( target-type : sources + : requirements * : target-name ? )
-{
- target-name ?= $(sources[1]:D=:S=) ;
-
- # Having periods (".") in the target name is problematic because the typed
- # generator will strip the suffix and use the bare name for the file
- # targets. Even though the location-prefix averts problems most times it
- # does not prevent ambiguity issues when referring to the test targets. For
- # example when using the XML log output. So we rename the target to remove
- # the periods, and provide an alias for users.
- local real-name = [ regex.replace $(target-name) "[.]" "~" ] ;
-
- local project = [ project.current ] ;
- # The <location-prefix> forces the build system for generate paths in the
- # form '$build_dir/array1.test/gcc/debug'. This is necessary to allow
- # post-processing tools to work.
- local t = [ targets.create-typed-target [ type.type-from-rule-name
- $(target-type) ] : $(project) : $(real-name) : $(sources) :
- $(requirements) <location-prefix>$(real-name).test ] ;
-
- # The alias to the real target, per period replacement above.
- if $(real-name) != $(target-name)
- {
- alias $(target-name) : $(t) ;
- }
-
- # Remember the test (for --dump-tests). A good way would be to collect all
- # given a project. This has some technical problems: e.g. we can not call
- # this dump from a Jamfile since projects referred by 'build-project' are
- # not available until the whole Jamfile has been loaded.
- .all-tests += $(t) ;
- return $(t) ;
-}
-
-
-# Note: passing more that one cpp file here is known to fail. Passing a cpp file
-# and a library target works.
-#
-rule compile ( sources + : requirements * : target-name ? )
-{
- return [ make-test compile : $(sources) : $(requirements) : $(target-name) ]
- ;
-}
-
-
-rule compile-fail ( sources + : requirements * : target-name ? )
-{
- return [ make-test compile-fail : $(sources) : $(requirements) :
- $(target-name) ] ;
-}
-
-
-rule link ( sources + : requirements * : target-name ? )
-{
- return [ make-test link : $(sources) : $(requirements) : $(target-name) ] ;
-}
-
-
-rule link-fail ( sources + : requirements * : target-name ? )
-{
- return [ make-test link-fail : $(sources) : $(requirements) : $(target-name)
- ] ;
-}
-
-
-rule handle-input-files ( input-files * )
-{
- if $(input-files[2])
- {
- # Check that sorting made when creating property-set instance will not
- # change the ordering.
- if [ sequence.insertion-sort $(input-files) ] != $(input-files)
- {
- errors.user-error "Names of input files must be sorted alphabetically"
- : "due to internal limitations" ;
- }
- }
- return <testing.input-file>$(input-files) ;
-}
-
-
-rule run ( sources + : args * : input-files * : requirements * : target-name ? :
- default-build * )
-{
- requirements += <testing.arg>$(args:J=" ") ;
- requirements += [ handle-input-files $(input-files) ] ;
- return [ make-test run : $(sources) : $(requirements) : $(target-name) ] ;
-}
-
-
-rule run-fail ( sources + : args * : input-files * : requirements * :
- target-name ? : default-build * )
-{
- requirements += <testing.arg>$(args:J=" ") ;
- requirements += [ handle-input-files $(input-files) ] ;
- return [ make-test run-fail : $(sources) : $(requirements) : $(target-name)
- ] ;
-}
-
-
-# Use 'test-suite' as a synonym for 'alias', for backward compatibility.
-IMPORT : alias : : test-suite ;
-
-
-# For all main targets in 'project-module', which are typed targets with type
-# derived from 'TEST', produce some interesting information.
-#
-rule dump-tests
-{
- for local t in $(.all-tests)
- {
- dump-test $(t) ;
- }
-}
-
-
-# Given a project location in normalized form (slashes are forward), compute the
-# name of the Boost library.
-#
-local rule get-library-name ( path )
-{
- # Path is in normalized form, so all slashes are forward.
- local match1 = [ MATCH /(tools|libs)/(.*)/(test|example) : $(path) ] ;
- local match2 = [ MATCH /(tools|libs)/(.*)$ : $(path) ] ;
- local match3 = [ MATCH (/status$) : $(path) ] ;
-
- if $(match1) { return $(match1[2]) ; }
- else if $(match2) { return $(match2[2]) ; }
- else if $(match3) { return "" ; }
- else if --dump-tests in [ modules.peek : ARGV ]
- {
- # The 'run' rule and others might be used outside boost. In that case,
- # just return the path, since the 'library name' makes no sense.
- return $(path) ;
- }
-}
-
-
-# Was an XML dump requested?
-.out-xml = [ MATCH --out-xml=(.*) : [ modules.peek : ARGV ] ] ;
-
-
-# Takes a target (instance of 'basic-target') and prints
-# - its type
-# - its name
-# - comments specified via the <test-info> property
-# - relative location of all source from the project root.
-#
-rule dump-test ( target )
-{
- local type = [ $(target).type ] ;
- local name = [ $(target).name ] ;
- local project = [ $(target).project ] ;
-
- local project-root = [ $(project).get project-root ] ;
- local library = [ get-library-name [ path.root [ $(project).get location ]
- [ path.pwd ] ] ] ;
- if $(library)
- {
- name = $(library)/$(name) ;
- }
-
- local sources = [ $(target).sources ] ;
- local source-files ;
- for local s in $(sources)
- {
- if [ class.is-a $(s) : file-reference ]
- {
- local location = [ path.root [ path.root [ $(s).name ]
- [ $(s).location ] ] [ path.pwd ] ] ;
-
- source-files += [ path.relative-to [ path.root $(project-root)
- [ path.pwd ] ] $(location) ] ;
- }
- }
-
- local target-name = [ $(project).get location ] // [ $(target).name ] .test
- ;
- target-name = $(target-name:J=) ;
-
- local r = [ $(target).requirements ] ;
- # Extract values of the <test-info> feature.
- local test-info = [ $(r).get <test-info> ] ;
-
- # If the user requested XML output on the command-line, add the test info to
- # that XML file rather than dumping them to stdout.
- if $(.out-xml)
- {
- local nl = "
-" ;
- .contents on $(.out-xml) +=
- "$(nl) <test type=\"$(type)\" name=\"$(name)\">"
- "$(nl) <target><![CDATA[$(target-name)]]></target>"
- "$(nl) <info><![CDATA[$(test-info)]]></info>"
- "$(nl) <source><![CDATA[$(source-files)]]></source>"
- "$(nl) </test>"
- ;
- }
- else
- {
- # Format them into a single string of quoted strings.
- test-info = \"$(test-info:J=\"\ \")\" ;
-
- ECHO boost-test($(type)) \"$(name)\" [$(test-info)] ":"
- \"$(source-files)\" ;
- }
-}
-
-
-# Register generators. Depending on target type, either 'expect-success' or
-# 'expect-failure' rule will be used.
-generators.register-standard testing.expect-success : OBJ : COMPILE ;
-generators.register-standard testing.expect-failure : OBJ : COMPILE_FAIL ;
-generators.register-standard testing.expect-success : RUN_OUTPUT : RUN ;
-generators.register-standard testing.expect-failure : RUN_OUTPUT : RUN_FAIL ;
-generators.register-standard testing.expect-failure : EXE : LINK_FAIL ;
-generators.register-standard testing.expect-success : EXE : LINK ;
-
-# Generator which runs an EXE and captures output.
-generators.register-standard testing.capture-output : EXE : RUN_OUTPUT ;
-
-# Generator which creates a target if sources run successfully. Differs from RUN
-# in that run output is not captured. The reason why it exists is that the 'run'
-# rule is much better for automated testing, but is not user-friendly (see
-# http://article.gmane.org/gmane.comp.lib.boost.build/6353).
-generators.register-standard testing.unit-test : EXE : UNIT_TEST ;
-
-
-# The action rules called by generators.
-
-# Causes the 'target' to exist after bjam invocation if and only if all the
-# dependencies were successfully built.
-#
-rule expect-success ( target : dependency + : requirements * )
-{
- **passed** $(target) : $(sources) ;
-}
-
-
-# Causes the 'target' to exist after bjam invocation if and only if all some of
-# the dependencies were not successfully built.
-#
-rule expect-failure ( target : dependency + : properties * )
-{
- local grist = [ MATCH ^<(.*)> : $(dependency:G) ] ;
- local marker = $(dependency:G=$(grist)*fail) ;
- (failed-as-expected) $(marker) ;
- FAIL_EXPECTED $(dependency) ;
- LOCATE on $(marker) = [ on $(dependency) return $(LOCATE) ] ;
- RMOLD $(marker) ;
- DEPENDS $(marker) : $(dependency) ;
- DEPENDS $(target) : $(marker) ;
- **passed** $(target) : $(marker) ;
-}
-
-
-# The rule/action combination used to report successful passing of a test.
-#
-rule **passed**
-{
- # Dump all the tests, if needed. We do it here, since dump should happen
- # only after all Jamfiles have been read, and there is no such place
- # currently defined (but there should be).
- if ! $(.dumped-tests) && ( --dump-tests in [ modules.peek : ARGV ] )
- {
- .dumped-tests = true ;
- dump-tests ;
- }
-
- # Force deletion of the target, in case any dependencies failed to build.
- RMOLD $(<) ;
-}
-
-
-# Used to create test files signifying passed tests.
-#
-actions **passed**
-{
- echo passed > "$(<)"
-}
-
-
-# Used to create replacement object files that do not get created during tests
-# that are expected to fail.
-#
-actions (failed-as-expected)
-{
- echo failed as expected > "$(<)"
-}
-
-
-rule run-path-setup ( target : source : properties * )
-{
- # For testing, we need to make sure that all dynamic libraries needed by the
- # test are found. So, we collect all paths from dependency libraries (via
- # xdll-path property) and add whatever explicit dll-path user has specified.
- # The resulting paths are added to the environment on each test invocation.
- local dll-paths = [ feature.get-values <dll-path> : $(properties) ] ;
- dll-paths += [ feature.get-values <xdll-path> : $(properties) ] ;
- dll-paths += [ on $(source) return $(RUN_PATH) ] ;
- dll-paths = [ sequence.unique $(dll-paths) ] ;
- if $(dll-paths)
- {
- dll-paths = [ sequence.transform path.native : $(dll-paths) ] ;
- PATH_SETUP on $(target) = [ common.prepend-path-variable-command
- [ os.shared-library-path-variable ] : $(dll-paths) ] ;
- }
-}
-
-
-local argv = [ modules.peek : ARGV ] ;
-
-toolset.flags testing.capture-output ARGS <testing.arg> ;
-toolset.flags testing.capture-output INPUT_FILES <testing.input-file> ;
-toolset.flags testing.capture-output LAUNCHER <testing.launcher> ;
-
-
-# Runs executable 'sources' and stores stdout in file 'target'. Unless
-# --preserve-test-targets command line option has been specified, removes the
-# executable. The 'target-to-remove' parameter controls what should be removed:
-# - if 'none', does not remove anything, ever
-# - if empty, removes 'source'
-# - if non-empty and not 'none', contains a list of sources to remove.
-#
-rule capture-output ( target : source : properties * : targets-to-remove * )
-{
- output-file on $(target) = $(target:S=.output) ;
- LOCATE on $(target:S=.output) = [ on $(target) return $(LOCATE) ] ;
-
- # The INCLUDES kill a warning about independent target...
- INCLUDES $(target) : $(target:S=.output) ;
- # but it also puts .output into dependency graph, so we must tell jam it is
- # OK if it cannot find the target or updating rule.
- NOCARE $(target:S=.output) ;
-
- # This has two-fold effect. First it adds input files to the dependendency
- # graph, preventing a warning. Second, it causes input files to be bound
- # before target is created. Therefore, they are bound using SEARCH setting
- # on them and not LOCATE setting of $(target), as in other case (due to jam
- # bug).
- DEPENDS $(target) : [ on $(target) return $(INPUT_FILES) ] ;
-
- if $(targets-to-remove) = none
- {
- targets-to-remove = ;
- }
- else if ! $(targets-to-remove)
- {
- targets-to-remove = $(source) ;
- }
-
- run-path-setup $(target) : $(source) : $(properties) ;
-
- if [ feature.get-values preserve-test-targets : $(properties) ] = off
- {
- TEMPORARY $(targets-to-remove) ;
- # Set a second action on target that will be executed after capture
- # output action. The 'RmTemps' rule has the 'ignore' modifier so it is
- # always considered succeeded. This is needed for 'run-fail' test. For
- # that test the target will be marked with FAIL_EXPECTED, and without
- # 'ignore' successful execution will be negated and be reported as
- # failure. With 'ignore' we do not detect a case where removing files
- # fails, but it is not likely to happen.
- RmTemps $(target) : $(targets-to-remove) ;
- }
-}
-
-
-if [ os.name ] = NT
-{
- .STATUS = %status% ;
- .SET_STATUS = "set status=%ERRORLEVEL%" ;
- .RUN_OUTPUT_NL = "echo." ;
- .STATUS_0 = "%status% EQU 0 (" ;
- .STATUS_NOT_0 = "%status% NEQ 0 (" ;
- .VERBOSE = "%verbose% EQU 1 (" ;
- .ENDIF = ")" ;
- .SHELL_SET = "set " ;
- .CATENATE = type ;
- .CP = copy ;
-}
-else
-{
- .STATUS = "$status" ;
- .SET_STATUS = "status=$?" ;
- .RUN_OUTPUT_NL = "echo" ;
- .STATUS_0 = "test $status -eq 0 ; then" ;
- .STATUS_NOT_0 = "test $status -ne 0 ; then" ;
- .VERBOSE = "test $verbose -eq 1 ; then" ;
- .ENDIF = "fi" ;
- .SHELL_SET = "" ;
- .CATENATE = cat ;
- .CP = cp ;
-}
-
-
-.VERBOSE_TEST = 0 ;
-if --verbose-test in [ modules.peek : ARGV ]
-{
- .VERBOSE_TEST = 1 ;
-}
-
-
-.RM = [ common.rm-command ] ;
-
-
-actions capture-output bind INPUT_FILES output-file
-{
- $(PATH_SETUP)
- $(LAUNCHER) "$(>)" $(ARGS) "$(INPUT_FILES)" > "$(output-file)" 2>&1
- $(.SET_STATUS)
- $(.RUN_OUTPUT_NL) >> "$(output-file)"
- echo EXIT STATUS: $(.STATUS) >> "$(output-file)"
- if $(.STATUS_0)
- $(.CP) "$(output-file)" "$(<)"
- $(.ENDIF)
- $(.SHELL_SET)verbose=$(.VERBOSE_TEST)
- if $(.STATUS_NOT_0)
- $(.SHELL_SET)verbose=1
- $(.ENDIF)
- if $(.VERBOSE)
- echo ====== BEGIN OUTPUT ======
- $(.CATENATE) "$(output-file)"
- echo ====== END OUTPUT ======
- $(.ENDIF)
- exit $(.STATUS)
-}
-
-
-actions quietly updated ignore piecemeal together RmTemps
-{
- $(.RM) "$(>)"
-}
-
-
-.MAKE_FILE = [ common.file-creation-command ] ;
-
-toolset.flags testing.unit-test LAUNCHER <testing.launcher> ;
-toolset.flags testing.unit-test ARGS <testing.arg> ;
-
-
-rule unit-test ( target : source : properties * )
-{
- run-path-setup $(target) : $(source) : $(properties) ;
-}
-
-
-actions unit-test
-{
- $(PATH_SETUP)
- $(LAUNCHER) "$(>)" $(ARGS) && $(.MAKE_FILE) "$(<)"
-}
-
-
-IMPORT $(__name__) : compile compile-fail run run-fail link link-fail
- : : compile compile-fail run run-fail link link-fail ;
-
-
-# This is a composing generator to support cases where a generator for the
-# specified target constructs other targets as well. One such example is msvc's
-# exe generator that constructs both EXE and PDB targets.
-type.register TIME : time ;
-generators.register-composing testing.time : : TIME ;
-
-
-# Note that this rule may be called multiple times for a single target in case
-# there are multiple actions operating on the same target in sequence. One such
-# example are msvc exe targets first created by a linker action and then updated
-# with an embedded manifest file by a separate action.
-rule record-time ( target : source : start end user system )
-{
- local src-string = [$(source:G=:J=",")"] " ;
- USER_TIME on $(target) += $(src-string)$(user) ;
- SYSTEM_TIME on $(target) += $(src-string)$(system) ;
-
- # We need the following variables because attempting to perform such
- # variable expansion in actions would not work due to quotes getting treated
- # as regular characters.
- USER_TIME_SECONDS on $(target) += $(src-string)$(user)" seconds" ;
- SYSTEM_TIME_SECONDS on $(target) += $(src-string)$(system)" seconds" ;
-}
-
-
-IMPORT testing : record-time : : testing.record-time ;
-
-
-# Calling this rule requests that Boost Build time how long it takes to build
-# the 'source' target and display the results both on the standard output and in
-# the 'target' file.
-#
-rule time ( target : sources + : properties * )
-{
- # Set up rule for recording timing information.
- __TIMING_RULE__ on $(sources) = testing.record-time $(target) ;
-
- # Make sure the sources get rebuilt any time we need to retrieve that
- # information.
- REBUILDS $(target) : $(sources) ;
-}
-
-
-actions time
-{
- echo user: $(USER_TIME)
- echo system: $(SYSTEM_TIME)
-
- echo user: $(USER_TIME_SECONDS) > "$(<)"
- echo system: $(SYSTEM_TIME_SECONDS) >> "$(<)"
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/tiff.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/tiff.jam b/ext/kenlm/jam-files/boost-build/tools/tiff.jam
deleted file mode 100644
index 14f235e..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/tiff.jam
+++ /dev/null
@@ -1,230 +0,0 @@
-# Copyright (c) 2010 Vladimir Prus.
-# Copyright (c) 2013 Steven Watanabe
-#
-# Use, modification and distribution is subject to the Boost Software
-# License Version 1.0. (See accompanying file LICENSE_1_0.txt or
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# Supports the libtiff library
-#
-# After 'using libtiff', the following targets are available:
-#
-# /libtiff//libtiff -- The libtiff library
-
-import project ;
-import ac ;
-import errors ;
-import "class" : new ;
-import targets ;
-import path ;
-import modules ;
-import errors ;
-import indirect ;
-import property ;
-import property-set ;
-
-header = tiff.h tiffio.hxx ;
-
-names = libtiff ;
-
-sources = tif_aux.c tif_close.c tif_codec.c tif_color.c tif_compress.c tif_dir.c tif_dirinfo.c
- tif_dirread.c tif_dirwrite.c tif_dumpmode.c tif_error.c tif_extension.c tif_fax3.c tif_fax3sm.c
- tif_getimage.c tif_jbig.c tif_jpeg.c tif_jpeg_12.c tif_ojpeg.c tif_flush.c tif_luv.c tif_lzw.c
- tif_next.c tif_open.c tif_packbits.c tif_pixarlog.c tif_predict.c tif_print.c tif_read.c tif_stream.cxx
- tif_swab.c tif_strip.c tif_thunder.c tif_tile.c tif_version.c tif_warning.c tif_write.c tif_zip.c ;
-
-library-id = 0 ;
-
-if --debug-configuration in [ modules.peek : ARGV ]
-{
- .debug = true ;
-}
-
-# Initializes the libtiff library.
-#
-# libtiff can be configured either to use pre-existing binaries
-# or to build the library from source.
-#
-# Options for configuring a prebuilt libtiff::
-#
-# <search>
-# The directory containing the libtiff binaries.
-# <name>
-# Overrides the default library name.
-# <include>
-# The directory containing the libtiff headers.
-#
-# If none of these options is specified, then the environmental
-# variables LIBTIFF_LIBRARY_PATH, LIBTIFF_NAME, and LIBTIFF_INCLUDE will
-# be used instead.
-#
-# Options for building libtiff from source::
-#
-# <source>
-# The libtiff source directory. Defaults to the environmental variable
-# LIBTIFF_SOURCE.
-# <tag>
-# A rule which computes the actual name of the compiled
-# libraries based on the build properties. Ignored
-# when using precompiled binaries.
-# <build-name>
-# The base name to use for the compiled library. Ignored
-# when using precompiled binaries.
-#
-# Examples::
-#
-# # Find libtiff in the default system location
-# using libtiff ;
-# # Build libtiff from source
-# using libtiff : 4.0.1 : <source>/home/steven/libtiff-4.0.1 ;
-# # Find libtiff in /usr/local
-# using libtiff : 4.0.1
-# : <include>/usr/local/include <search>/usr/local/lib ;
-# # Build libtiff from source for msvc and find
-# # prebuilt binaries for gcc.
-# using libtiff : 4.0.1 : <source>C:/Devel/src/libtiff-4.0.1 : <toolset>msvc ;
-# using libtiff : 4.0.1 : : <toolset>gcc ;
-#
-rule init (
- version ?
- # The libtiff version (currently ignored)
-
- : options *
- # A list of the options to use
-
- : requirements *
- # The requirements for the libtiff target
-
- : is-default ?
- # Default configurations are only used when libtiff
- # has not yet been configured.
- )
-{
- local caller = [ project.current ] ;
-
- if ! $(.initialized)
- {
- .initialized = true ;
-
- project.initialize $(__name__) ;
- .project = [ project.current ] ;
- project libtiff ;
- }
-
- local library-path = [ property.select <search> : $(options) ] ;
- library-path = $(library-path:G=) ;
- local include-path = [ property.select <include> : $(options) ] ;
- include-path = $(include-path:G=) ;
- local source-path = [ property.select <source> : $(options) ] ;
- source-path = $(source-path:G=) ;
- local library-name = [ property.select <name> : $(options) ] ;
- library-name = $(library-name:G=) ;
- local tag = [ property.select <tag> : $(options) ] ;
- tag = $(tag:G=) ;
- local build-name = [ property.select <build-name> : $(options) ] ;
- build-name = $(build-name:G=) ;
-
- condition = [ property-set.create $(requirements) ] ;
- condition = [ property-set.create [ $(condition).base ] ] ;
-
- local no-build-from-source ;
- # Ignore environmental ZLIB_SOURCE if this initialization
- # requested to search for a specific pre-built library.
- if $(library-path) || $(include-path) || $(library-name)
- {
- if $(source-path) || $(tag) || $(build-name)
- {
- errors.user-error "incompatible options for libtiff:"
- [ property.select <search> <include> <name> : $(options) ] "and"
- [ property.select <source> <tag> <build-name> : $(options) ] ;
- }
- else
- {
- no-build-from-source = true ;
- }
- }
-
- source-path ?= [ modules.peek : ZLIB_SOURCE ] ;
-
- if $(.configured.$(condition))
- {
- if $(is-default)
- {
- if $(.debug)
- {
- ECHO "notice: [libtiff] libtiff is already configured" ;
- }
- }
- else
- {
- errors.user-error "libtiff is already configured" ;
- }
- return ;
- }
- else if $(source-path) && ! $(no-build-from-source)
- {
- build-name ?= z ;
- library-id = [ CALC $(library-id) + 1 ] ;
- tag = [ MATCH ^@?(.*)$ : $(tag) ] ;
- if $(tag) && ! [ MATCH ^([^%]*)%([^%]+)$ : $(tag) ]
- {
- tag = [ indirect.make $(tag) : [ $(caller).project-module ] ] ;
- }
- sources = [ path.glob $(source-path) : $(sources) ] ;
- if $(.debug)
- {
- ECHO "notice: [libtiff] Building libtiff from source as $(build-name)" ;
- if $(condition)
- {
- ECHO "notice: [libtiff] Condition" [ $(condition).raw ] ;
- }
- if $(sources)
- {
- ECHO "notice: [libtiff] found libtiff source in $(source-path)" ;
- }
- else
- {
- ECHO "warning: [libtiff] could not find libtiff source in $(source-path)" ;
- }
- }
- local target ;
- if $(sources) {
- target = [ targets.create-typed-target LIB : $(.project)
- : $(build-name).$(library-id)
- : $(sources)
- : $(requirements)
- <tag>@$(tag)
- <include>$(source-path)
- <toolset>msvc:<define>_CRT_SECURE_NO_DEPRECATE
- <toolset>msvc:<define>_SCL_SECURE_NO_DEPRECATE
- <link>shared:<define>ZLIB_DLL
- :
- : <include>$(source-path) ] ;
- }
-
- local mt = [ new ac-library libtiff : $(.project) : $(condition) ] ;
- $(mt).set-header $(header) ;
- $(mt).set-default-names $(names) ;
- if $(target)
- {
- $(mt).set-target $(target) ;
- }
- targets.main-target-alternative $(mt) ;
- } else {
- if $(.debug)
- {
- ECHO "notice: [libtiff] Using pre-installed library" ;
- if $(condition)
- {
- ECHO "notice: [libtiff] Condition" [ $(condition).raw ] ;
- }
- }
-
- local mt = [ new ac-library libtiff : $(.project) : $(condition) :
- $(include-path) : $(library-path) : $(library-name) : $(root) ] ;
- $(mt).set-header $(header) ;
- $(mt).set-default-names $(names) ;
- targets.main-target-alternative $(mt) ;
- }
- .configured.$(condition) = true ;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/types/asm.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/types/asm.jam b/ext/kenlm/jam-files/boost-build/tools/types/asm.jam
deleted file mode 100644
index a340db3..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/types/asm.jam
+++ /dev/null
@@ -1,4 +0,0 @@
-# Copyright Craig Rodrigues 2005. Distributed under the Boost
-# Software License, Version 1.0. (See accompanying
-# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
-type ASM : s S asm ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/types/cpp.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/types/cpp.jam b/ext/kenlm/jam-files/boost-build/tools/types/cpp.jam
deleted file mode 100644
index 29d8faa..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/types/cpp.jam
+++ /dev/null
@@ -1,90 +0,0 @@
-# Copyright 2004 David Abrahams
-# Copyright 2002, 2003, 2004, 2005, 2006 Vladimir Prus
-# Copyright 2010 Rene Rivera
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-import scanner ;
-import type ;
-
-
-class c-scanner : scanner
-{
- import path ;
- import regex ;
- import scanner ;
- import sequence ;
- import virtual-target ;
-
- rule __init__ ( includes * )
- {
- scanner.__init__ ;
-
- for local i in $(includes)
- {
- self.includes += [ sequence.transform path.native : [ regex.split
- $(i:G=) "&&" ] ] ;
- }
- }
-
- rule pattern ( )
- {
- return "#[ \t]*include[ ]*(<(.*)>|\"(.*)\")" ;
- }
-
- rule process ( target : matches * : binding )
- {
- local angle = [ regex.transform $(matches) : "<(.*)>" ] ;
- angle = [ sequence.transform path.native : $(angle) ] ;
- local quoted = [ regex.transform $(matches) : "\"(.*)\"" ] ;
- quoted = [ sequence.transform path.native : $(quoted) ] ;
-
- # CONSIDER: the new scoping rules seem to defeat "on target" variables.
- local g = [ on $(target) return $(HDRGRIST) ] ;
- local b = [ NORMALIZE_PATH $(binding:D) ] ;
-
- # Attach binding of including file to included targets. When a target is
- # directly created from a virtual target this extra information is
- # unnecessary. But in other cases, it allows us to distinguish between
- # two headers of the same name included from different places. We do not
- # need this extra information for angle includes, since they should not
- # depend on the including file (we can not get literal "." in the
- # include path).
- local g2 = $(g)"#"$(b) ;
-
- angle = $(angle:G=$(g)) ;
- quoted = $(quoted:G=$(g2)) ;
-
- local all = $(angle) $(quoted) ;
-
- INCLUDES $(target) : $(all) ;
- NOCARE $(all) ;
- SEARCH on $(angle) = $(self.includes:G=) ;
- SEARCH on $(quoted) = $(b) $(self.includes:G=) ;
-
- # Just propagate the current scanner to includes, in hope that includes
- # do not change scanners.
- scanner.propagate $(__name__) : $(all) : $(target) ;
-
- ISFILE $(all) ;
- }
-}
-
-scanner.register c-scanner : include ;
-
-type.register CPP : cpp cxx cc ;
-type.register H : h ;
-type.register HPP : hpp : H ;
-type.register C : c ;
-
-# It most cases where a CPP file or a H file is a source of some action, we
-# should rebuild the result if any of files included by CPP/H are changed. One
-# case when this is not needed is installation, which is handled specifically.
-type.set-scanner CPP : c-scanner ;
-type.set-scanner C : c-scanner ;
-# One case where scanning of H/HPP files is necessary is PCH generation -- if
-# any header included by HPP being precompiled changes, we need to recompile the
-# header.
-type.set-scanner H : c-scanner ;
-type.set-scanner HPP : c-scanner ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/types/exe.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/types/exe.jam b/ext/kenlm/jam-files/boost-build/tools/types/exe.jam
deleted file mode 100644
index 4710951..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/types/exe.jam
+++ /dev/null
@@ -1,9 +0,0 @@
-# Copyright David Abrahams 2004. Distributed under the Boost
-# Software License, Version 1.0. (See accompanying
-# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
-
-import type ;
-
-type.register EXE ;
-type.set-generated-target-suffix EXE : <target-os>windows : "exe" ;
-type.set-generated-target-suffix EXE : <target-os>cygwin : "exe" ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/types/html.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/types/html.jam b/ext/kenlm/jam-files/boost-build/tools/types/html.jam
deleted file mode 100644
index 5cd337d..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/types/html.jam
+++ /dev/null
@@ -1,4 +0,0 @@
-# Copyright David Abrahams 2004. Distributed under the Boost
-# Software License, Version 1.0. (See accompanying
-# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
-type HTML : html ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/types/lib.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/types/lib.jam b/ext/kenlm/jam-files/boost-build/tools/types/lib.jam
deleted file mode 100644
index 854ab8f..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/types/lib.jam
+++ /dev/null
@@ -1,74 +0,0 @@
-# Copyright David Abrahams 2004. Distributed under the Boost
-# Software License, Version 1.0. (See accompanying
-# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
-
-import type ; # for set-generated-target-suffix
-import os ;
-
-# The following naming scheme is used for libraries.
-#
-# On *nix:
-# libxxx.a static library
-# libxxx.so shared library
-#
-# On windows (msvc)
-# libxxx.lib static library
-# xxx.dll DLL
-# xxx.lib import library
-#
-# On windows (mingw):
-# libxxx.a static library
-# libxxx.dll DLL
-# libxxx.dll.a import library
-#
-# On cygwin i.e. <target-os>cygwin
-# libxxx.a static library
-# cygxxx.dll DLL
-# libxxx.dll.a import library
-#
-
-type.register LIB ;
-
-# FIXME: should not register both extensions on both platforms.
-type.register STATIC_LIB : a lib : LIB ;
-
-# The 'lib' prefix is used everywhere
-type.set-generated-target-prefix STATIC_LIB : : lib ;
-
-# Use '.lib' suffix for windows
-type.set-generated-target-suffix STATIC_LIB : <target-os>windows : lib ;
-
-# Except with gcc.
-type.set-generated-target-suffix STATIC_LIB : <toolset>gcc <target-os>windows : a ;
-
-# Use xxx.lib for import libs
-type IMPORT_LIB : : STATIC_LIB ;
-type.set-generated-target-prefix IMPORT_LIB : : "" ;
-type.set-generated-target-suffix IMPORT_LIB : : lib ;
-
-# Except with gcc (mingw or cygwin), where use libxxx.dll.a
-type.set-generated-target-prefix IMPORT_LIB : <toolset>gcc : lib ;
-type.set-generated-target-suffix IMPORT_LIB : <toolset>gcc : dll.a ;
-
-type.register SHARED_LIB : so dll dylib : LIB ;
-
-# Both mingw and cygwin use libxxx.dll naming scheme.
-# On Linux, use "lib" prefix
-type.set-generated-target-prefix SHARED_LIB : : lib ;
-# But don't use it on windows
-type.set-generated-target-prefix SHARED_LIB : <target-os>windows : "" ;
-# But use it again on mingw
-type.set-generated-target-prefix SHARED_LIB : <toolset>gcc <target-os>windows : lib ;
-# And use 'cyg' on cygwin
-type.set-generated-target-prefix SHARED_LIB : <target-os>cygwin : cyg ;
-
-
-type.set-generated-target-suffix SHARED_LIB : <target-os>windows : dll ;
-type.set-generated-target-suffix SHARED_LIB : <target-os>cygwin : dll ;
-type.set-generated-target-suffix SHARED_LIB : <target-os>darwin : dylib ;
-
-type SEARCHED_LIB : : LIB ;
-# This is needed so that when we create a target of SEARCHED_LIB
-# type, there's no prefix or suffix automatically added.
-type.set-generated-target-prefix SEARCHED_LIB : : "" ;
-type.set-generated-target-suffix SEARCHED_LIB : : "" ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/types/obj.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/types/obj.jam b/ext/kenlm/jam-files/boost-build/tools/types/obj.jam
deleted file mode 100644
index 6afbcaa..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/types/obj.jam
+++ /dev/null
@@ -1,9 +0,0 @@
-# Copyright David Abrahams 2004. Distributed under the Boost
-# Software License, Version 1.0. (See accompanying
-# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
-
-import type ;
-
-type.register OBJ : o obj ;
-type.set-generated-target-suffix OBJ : <target-os>windows : obj ;
-type.set-generated-target-suffix OBJ : <target-os>cygwin : obj ;
[17/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/regexp.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/regexp.c b/ext/kenlm/jam-files/engine/regexp.c
deleted file mode 100644
index c64201b..0000000
--- a/ext/kenlm/jam-files/engine/regexp.c
+++ /dev/null
@@ -1,1329 +0,0 @@
-/*
- * regcomp and regexec -- regsub and regerror are elsewhere
- *
- * Copyright (c) 1986 by University of Toronto.
- * Written by Henry Spencer. Not derived from licensed software.
- *
- * Permission is granted to anyone to use this software for any
- * purpose on any computer system, and to redistribute it freely,
- * subject to the following restrictions:
- *
- * 1. The author is not responsible for the consequences of use of
- * this software, no matter how awful, even if they arise
- * from defects in it.
- *
- * 2. The origin of this software must not be misrepresented, either
- * by explicit claim or by omission.
- *
- * 3. Altered versions must be plainly marked as such, and must not
- * be misrepresented as being the original software.
- *** THIS IS AN ALTERED VERSION. It was altered by John Gilmore,
- *** hoptoad!gnu, on 27 Dec 1986, to add \n as an alternative to |
- *** to assist in implementing egrep.
- *** THIS IS AN ALTERED VERSION. It was altered by John Gilmore,
- *** hoptoad!gnu, on 27 Dec 1986, to add \< and \> for word-matching
- *** as in BSD grep and ex.
- *** THIS IS AN ALTERED VERSION. It was altered by John Gilmore,
- *** hoptoad!gnu, on 28 Dec 1986, to optimize characters quoted with \.
- *** THIS IS AN ALTERED VERSION. It was altered by James A. Woods,
- *** ames!jaw, on 19 June 1987, to quash a regcomp() redundancy.
- *** THIS IS AN ALTERED VERSION. It was altered by Christopher Seiwald
- *** seiwald@vix.com, on 28 August 1993, for use in jam. Regmagic.h
- *** was moved into regexp.h, and the include of regexp.h now uses "'s
- *** to avoid conflicting with the system regexp.h. Const, bless its
- *** soul, was removed so it can compile everywhere. The declaration
- *** of strchr() was in conflict on AIX, so it was removed (as it is
- *** happily defined in string.h).
- *** THIS IS AN ALTERED VERSION. It was altered by Christopher Seiwald
- *** seiwald@perforce.com, on 20 January 2000, to use function prototypes.
- *
- * Beware that some of this code is subtly aware of the way operator precedence
- * is structured in regular expressions. Serious changes in regular-expression
- * syntax might require a total rethink.
- */
-
-
-#include "jam.h"
-#include "regexp.h"
-
-#include <stdio.h>
-#include <ctype.h>
-#ifndef ultrix
-# include <stdlib.h>
-#endif
-#include <string.h>
-
-
-/*
- * The "internal use only" fields in regexp.h are present to pass info from
- * compile to execute that permits the execute phase to run lots faster on
- * simple cases. They are:
- :
- * regstart char that must begin a match; '\0' if none obvious.
- * reganch is the match anchored (at beginning-of-line only)?
- * regmust string (pointer into program) that match must include, or NULL.
- * regmlen length of regmust string.
- *
- * Regstart and reganch permit very fast decisions on suitable starting points
- * for a match, cutting down the work a lot. Regmust permits fast rejection of
- * lines that cannot possibly match. The regmust tests are costly enough that
- * regcomp() supplies a regmust only if the r.e. contains something potentially
- * expensive (at present, the only such thing detected is * or + at the start of
- * the r.e., which can involve a lot of backup). Regmlen is supplied because the
- * test in regexec() needs it and regcomp() is computing it anyway.
- */
-
-/*
- * Structure for regexp "program". This is essentially a linear encoding of a
- * nondeterministic finite-state machine (aka syntax charts or "railroad normal
- * form" in parsing technology). Each node is an opcode plus a "next" pointer,
- * possibly plus an operand. "Next" pointers of all nodes except BRANCH
- * implement concatenation; a "next" pointer with a BRANCH on both ends of it is
- * connecting two alternatives. [Here we have one of the subtle syntax
- * dependencies: an individual BRANCH, as opposed to a collection of them, is
- * never concatenated with anything because of operator precedence.] The operand
- * of some types of node is a literal string; for others, it is a node leading
- * into a sub-FSM. In particular, the operand of a BRANCH node is the first node
- * of the branch. [NB this is *not* a tree structure: the tail of the branch
- * connects to the thing following the set of BRANCHes.] The opcodes are:
- */
-
-/* definition number opnd? meaning */
-#define END 0 /* no End of program. */
-#define BOL 1 /* no Match "" at beginning of line. */
-#define EOL 2 /* no Match "" at end of line. */
-#define ANY 3 /* no Match any one character. */
-#define ANYOF 4 /* str Match any character in this string. */
-#define ANYBUT 5 /* str Match any character not in this string. */
-#define BRANCH 6 /* node Match this alternative, or the next... */
-#define BACK 7 /* no Match "", "next" ptr points backward. */
-#define EXACTLY 8 /* str Match this string. */
-#define NOTHING 9 /* no Match empty string. */
-#define STAR 10 /* node Match this (simple) thing 0 or more times. */
-#define PLUS 11 /* node Match this (simple) thing 1 or more times. */
-#define WORDA 12 /* no Match "" at wordchar, where prev is nonword */
-#define WORDZ 13 /* no Match "" at nonwordchar, where prev is word */
-#define OPEN 20 /* no Mark this point in input as start of #n. */
- /* OPEN+1 is number 1, etc. */
-#define CLOSE 30 /* no Analogous to OPEN. */
-
-
-/*
- * Opcode notes:
- *
- * BRANCH The set of branches constituting a single choice are hooked
- * together with their "next" pointers, since precedence prevents
- * anything being concatenated to any individual branch. The
- * "next" pointer of the last BRANCH in a choice points to the
- * thing following the whole choice. This is also where the
- * final "next" pointer of each individual branch points; each
- * branch starts with the operand node of a BRANCH node.
- *
- * BACK Normal "next" pointers all implicitly point forward; BACK
- * exists to make loop structures possible.
- *
- * STAR,PLUS '?', and complex '*' and '+', are implemented as circular
- * BRANCH structures using BACK. Simple cases (one character
- * per match) are implemented with STAR and PLUS for speed
- * and to minimize recursive plunges.
- *
- * OPEN,CLOSE ...are numbered at compile time.
- */
-
-/*
- * A node is one char of opcode followed by two chars of "next" pointer.
- * "Next" pointers are stored as two 8-bit pieces, high order first. The
- * value is a positive offset from the opcode of the node containing it.
- * An operand, if any, simply follows the node. (Note that much of the
- * code generation knows about this implicit relationship.)
- *
- * Using two bytes for the "next" pointer is vast overkill for most things,
- * but allows patterns to get big without disasters.
- */
-#define OP(p) (*(p))
-#define NEXT(p) (((*((p)+1)&0377)<<8) + (*((p)+2)&0377))
-#define OPERAND(p) ((p) + 3)
-
-/*
- * See regmagic.h for one further detail of program structure.
- */
-
-
-/*
- * Utility definitions.
- */
-#ifndef CHARBITS
-#define UCHARAT(p) ((int)*(const unsigned char *)(p))
-#else
-#define UCHARAT(p) ((int)*(p)&CHARBITS)
-#endif
-
-#define FAIL(m) { regerror(m); return(NULL); }
-#define ISMULT(c) ((c) == '*' || (c) == '+' || (c) == '?')
-
-/*
- * Flags to be passed up and down.
- */
-#define HASWIDTH 01 /* Known never to match null string. */
-#define SIMPLE 02 /* Simple enough to be STAR/PLUS operand. */
-#define SPSTART 04 /* Starts with * or +. */
-#define WORST 0 /* Worst case. */
-
-/*
- * Global work variables for regcomp().
- */
-static char *regparse; /* Input-scan pointer. */
-static int regnpar; /* () count. */
-static char regdummy;
-static char *regcode; /* Code-emit pointer; ®dummy = don't. */
-static long regsize; /* Code size. */
-
-/*
- * Forward declarations for regcomp()'s friends.
- */
-#ifndef STATIC
-#define STATIC static
-#endif
-STATIC char *reg( int paren, int *flagp );
-STATIC char *regbranch( int *flagp );
-STATIC char *regpiece( int *flagp );
-STATIC char *regatom( int *flagp );
-STATIC char *regnode( int op );
-STATIC char *regnext( register char *p );
-STATIC void regc( int b );
-STATIC void reginsert( char op, char *opnd );
-STATIC void regtail( char *p, char *val );
-STATIC void regoptail( char *p, char *val );
-#ifdef STRCSPN
-STATIC int strcspn();
-#endif
-
-/*
- - regcomp - compile a regular expression into internal code
- *
- * We can't allocate space until we know how big the compiled form will be,
- * but we can't compile it (and thus know how big it is) until we've got a
- * place to put the code. So we cheat: we compile it twice, once with code
- * generation turned off and size counting turned on, and once "for real".
- * This also means that we don't allocate space until we are sure that the
- * thing really will compile successfully, and we never have to move the
- * code and thus invalidate pointers into it. (Note that it has to be in
- * one piece because free() must be able to free it all.)
- *
- * Beware that the optimization-preparation code in here knows about some
- * of the structure of the compiled regexp.
- */
-regexp *
-regcomp( const char *exp )
-{
- register regexp *r;
- register char *scan;
- register char *longest;
- register unsigned len;
- int flags;
-
- if (exp == NULL)
- FAIL("NULL argument");
-
- /* First pass: determine size, legality. */
-#ifdef notdef
- if (exp[0] == '.' && exp[1] == '*') exp += 2; /* aid grep */
-#endif
- regparse = (char *)exp;
- regnpar = 1;
- regsize = 0L;
- regcode = ®dummy;
- regc(MAGIC);
- if (reg(0, &flags) == NULL)
- return(NULL);
-
- /* Small enough for pointer-storage convention? */
- if (regsize >= 32767L) /* Probably could be 65535L. */
- FAIL("regexp too big");
-
- /* Allocate space. */
- r = (regexp *)BJAM_MALLOC(sizeof(regexp) + (unsigned)regsize);
- if (r == NULL)
- FAIL("out of space");
-
- /* Second pass: emit code. */
- regparse = (char *)exp;
- regnpar = 1;
- regcode = r->program;
- regc(MAGIC);
- if (reg(0, &flags) == NULL)
- return(NULL);
-
- /* Dig out information for optimizations. */
- r->regstart = '\0'; /* Worst-case defaults. */
- r->reganch = 0;
- r->regmust = NULL;
- r->regmlen = 0;
- scan = r->program+1; /* First BRANCH. */
- if (OP(regnext(scan)) == END) { /* Only one top-level choice. */
- scan = OPERAND(scan);
-
- /* Starting-point info. */
- if (OP(scan) == EXACTLY)
- r->regstart = *OPERAND(scan);
- else if (OP(scan) == BOL)
- r->reganch++;
-
- /*
- * If there's something expensive in the r.e., find the
- * longest literal string that must appear and make it the
- * regmust. Resolve ties in favor of later strings, since
- * the regstart check works with the beginning of the r.e.
- * and avoiding duplication strengthens checking. Not a
- * strong reason, but sufficient in the absence of others.
- */
- if (flags&SPSTART) {
- longest = NULL;
- len = 0;
- for (; scan != NULL; scan = regnext(scan))
- if (OP(scan) == EXACTLY && strlen(OPERAND(scan)) >= len) {
- longest = OPERAND(scan);
- len = strlen(OPERAND(scan));
- }
- r->regmust = longest;
- r->regmlen = len;
- }
- }
-
- return(r);
-}
-
-/*
- - reg - regular expression, i.e. main body or parenthesized thing
- *
- * Caller must absorb opening parenthesis.
- *
- * Combining parenthesis handling with the base level of regular expression
- * is a trifle forced, but the need to tie the tails of the branches to what
- * follows makes it hard to avoid.
- */
-static char *
-reg(
- int paren, /* Parenthesized? */
- int *flagp )
-{
- register char *ret;
- register char *br;
- register char *ender;
- register int parno = 0;
- int flags;
-
- *flagp = HASWIDTH; /* Tentatively. */
-
- /* Make an OPEN node, if parenthesized. */
- if (paren) {
- if (regnpar >= NSUBEXP)
- FAIL("too many ()");
- parno = regnpar;
- regnpar++;
- ret = regnode(OPEN+parno);
- } else
- ret = NULL;
-
- /* Pick up the branches, linking them together. */
- br = regbranch(&flags);
- if (br == NULL)
- return(NULL);
- if (ret != NULL)
- regtail(ret, br); /* OPEN -> first. */
- else
- ret = br;
- if (!(flags&HASWIDTH))
- *flagp &= ~HASWIDTH;
- *flagp |= flags&SPSTART;
- while (*regparse == '|' || *regparse == '\n') {
- regparse++;
- br = regbranch(&flags);
- if (br == NULL)
- return(NULL);
- regtail(ret, br); /* BRANCH -> BRANCH. */
- if (!(flags&HASWIDTH))
- *flagp &= ~HASWIDTH;
- *flagp |= flags&SPSTART;
- }
-
- /* Make a closing node, and hook it on the end. */
- ender = regnode((paren) ? CLOSE+parno : END);
- regtail(ret, ender);
-
- /* Hook the tails of the branches to the closing node. */
- for (br = ret; br != NULL; br = regnext(br))
- regoptail(br, ender);
-
- /* Check for proper termination. */
- if (paren && *regparse++ != ')') {
- FAIL("unmatched ()");
- } else if (!paren && *regparse != '\0') {
- if (*regparse == ')') {
- FAIL("unmatched ()");
- } else
- FAIL("junk on end"); /* "Can't happen". */
- /* NOTREACHED */
- }
-
- return(ret);
-}
-
-/*
- - regbranch - one alternative of an | operator
- *
- * Implements the concatenation operator.
- */
-static char *
-regbranch( int *flagp )
-{
- register char *ret;
- register char *chain;
- register char *latest;
- int flags;
-
- *flagp = WORST; /* Tentatively. */
-
- ret = regnode(BRANCH);
- chain = NULL;
- while (*regparse != '\0' && *regparse != ')' &&
- *regparse != '\n' && *regparse != '|') {
- latest = regpiece(&flags);
- if (latest == NULL)
- return(NULL);
- *flagp |= flags&HASWIDTH;
- if (chain == NULL) /* First piece. */
- *flagp |= flags&SPSTART;
- else
- regtail(chain, latest);
- chain = latest;
- }
- if (chain == NULL) /* Loop ran zero times. */
- (void) regnode(NOTHING);
-
- return(ret);
-}
-
-/*
- - regpiece - something followed by possible [*+?]
- *
- * Note that the branching code sequences used for ? and the general cases
- * of * and + are somewhat optimized: they use the same NOTHING node as
- * both the endmarker for their branch list and the body of the last branch.
- * It might seem that this node could be dispensed with entirely, but the
- * endmarker role is not redundant.
- */
-static char *
-regpiece( int *flagp )
-{
- register char *ret;
- register char op;
- register char *next;
- int flags;
-
- ret = regatom(&flags);
- if (ret == NULL)
- return(NULL);
-
- op = *regparse;
- if (!ISMULT(op)) {
- *flagp = flags;
- return(ret);
- }
-
- if (!(flags&HASWIDTH) && op != '?')
- FAIL("*+ operand could be empty");
- *flagp = (op != '+') ? (WORST|SPSTART) : (WORST|HASWIDTH);
-
- if (op == '*' && (flags&SIMPLE))
- reginsert(STAR, ret);
- else if (op == '*') {
- /* Emit x* as (x&|), where & means "self". */
- reginsert(BRANCH, ret); /* Either x */
- regoptail(ret, regnode(BACK)); /* and loop */
- regoptail(ret, ret); /* back */
- regtail(ret, regnode(BRANCH)); /* or */
- regtail(ret, regnode(NOTHING)); /* null. */
- } else if (op == '+' && (flags&SIMPLE))
- reginsert(PLUS, ret);
- else if (op == '+') {
- /* Emit x+ as x(&|), where & means "self". */
- next = regnode(BRANCH); /* Either */
- regtail(ret, next);
- regtail(regnode(BACK), ret); /* loop back */
- regtail(next, regnode(BRANCH)); /* or */
- regtail(ret, regnode(NOTHING)); /* null. */
- } else if (op == '?') {
- /* Emit x? as (x|) */
- reginsert(BRANCH, ret); /* Either x */
- regtail(ret, regnode(BRANCH)); /* or */
- next = regnode(NOTHING); /* null. */
- regtail(ret, next);
- regoptail(ret, next);
- }
- regparse++;
- if (ISMULT(*regparse))
- FAIL("nested *?+");
-
- return(ret);
-}
-
-/*
- - regatom - the lowest level
- *
- * Optimization: gobbles an entire sequence of ordinary characters so that
- * it can turn them into a single node, which is smaller to store and
- * faster to run. Backslashed characters are exceptions, each becoming a
- * separate node; the code is simpler that way and it's not worth fixing.
- */
-static char *
-regatom( int *flagp )
-{
- register char *ret;
- int flags;
-
- *flagp = WORST; /* Tentatively. */
-
- switch (*regparse++) {
- /* FIXME: these chars only have meaning at beg/end of pat? */
- case '^':
- ret = regnode(BOL);
- break;
- case '$':
- ret = regnode(EOL);
- break;
- case '.':
- ret = regnode(ANY);
- *flagp |= HASWIDTH|SIMPLE;
- break;
- case '[': {
- register int classr;
- register int classend;
-
- if (*regparse == '^') { /* Complement of range. */
- ret = regnode(ANYBUT);
- regparse++;
- } else
- ret = regnode(ANYOF);
- if (*regparse == ']' || *regparse == '-')
- regc(*regparse++);
- while (*regparse != '\0' && *regparse != ']') {
- if (*regparse == '-') {
- regparse++;
- if (*regparse == ']' || *regparse == '\0')
- regc('-');
- else {
- classr = UCHARAT(regparse-2)+1;
- classend = UCHARAT(regparse);
- if (classr > classend+1)
- FAIL("invalid [] range");
- for (; classr <= classend; classr++)
- regc(classr);
- regparse++;
- }
- } else
- regc(*regparse++);
- }
- regc('\0');
- if (*regparse != ']')
- FAIL("unmatched []");
- regparse++;
- *flagp |= HASWIDTH|SIMPLE;
- }
- break;
- case '(':
- ret = reg(1, &flags);
- if (ret == NULL)
- return(NULL);
- *flagp |= flags&(HASWIDTH|SPSTART);
- break;
- case '\0':
- case '|':
- case '\n':
- case ')':
- FAIL("internal urp"); /* Supposed to be caught earlier. */
- break;
- case '?':
- case '+':
- case '*':
- FAIL("?+* follows nothing");
- break;
- case '\\':
- switch (*regparse++) {
- case '\0':
- FAIL("trailing \\");
- break;
- case '<':
- ret = regnode(WORDA);
- break;
- case '>':
- ret = regnode(WORDZ);
- break;
- /* FIXME: Someday handle \1, \2, ... */
- default:
- /* Handle general quoted chars in exact-match routine */
- goto de_fault;
- }
- break;
- de_fault:
- default:
- /*
- * Encode a string of characters to be matched exactly.
- *
- * This is a bit tricky due to quoted chars and due to
- * '*', '+', and '?' taking the SINGLE char previous
- * as their operand.
- *
- * On entry, the char at regparse[-1] is going to go
- * into the string, no matter what it is. (It could be
- * following a \ if we are entered from the '\' case.)
- *
- * Basic idea is to pick up a good char in ch and
- * examine the next char. If it's *+? then we twiddle.
- * If it's \ then we frozzle. If it's other magic char
- * we push ch and terminate the string. If none of the
- * above, we push ch on the string and go around again.
- *
- * regprev is used to remember where "the current char"
- * starts in the string, if due to a *+? we need to back
- * up and put the current char in a separate, 1-char, string.
- * When regprev is NULL, ch is the only char in the
- * string; this is used in *+? handling, and in setting
- * flags |= SIMPLE at the end.
- */
- {
- char *regprev;
- register char ch;
-
- regparse--; /* Look at cur char */
- ret = regnode(EXACTLY);
- for ( regprev = 0 ; ; ) {
- ch = *regparse++; /* Get current char */
- switch (*regparse) { /* look at next one */
-
- default:
- regc(ch); /* Add cur to string */
- break;
-
- case '.': case '[': case '(':
- case ')': case '|': case '\n':
- case '$': case '^':
- case '\0':
- /* FIXME, $ and ^ should not always be magic */
- magic:
- regc(ch); /* dump cur char */
- goto done; /* and we are done */
-
- case '?': case '+': case '*':
- if (!regprev) /* If just ch in str, */
- goto magic; /* use it */
- /* End mult-char string one early */
- regparse = regprev; /* Back up parse */
- goto done;
-
- case '\\':
- regc(ch); /* Cur char OK */
- switch (regparse[1]){ /* Look after \ */
- case '\0':
- case '<':
- case '>':
- /* FIXME: Someday handle \1, \2, ... */
- goto done; /* Not quoted */
- default:
- /* Backup point is \, scan * point is after it. */
- regprev = regparse;
- regparse++;
- continue; /* NOT break; */
- }
- }
- regprev = regparse; /* Set backup point */
- }
- done:
- regc('\0');
- *flagp |= HASWIDTH;
- if (!regprev) /* One char? */
- *flagp |= SIMPLE;
- }
- break;
- }
-
- return(ret);
-}
-
-/*
- - regnode - emit a node
- */
-static char * /* Location. */
-regnode( int op )
-{
- register char *ret;
- register char *ptr;
-
- ret = regcode;
- if (ret == ®dummy) {
- regsize += 3;
- return(ret);
- }
-
- ptr = ret;
- *ptr++ = op;
- *ptr++ = '\0'; /* Null "next" pointer. */
- *ptr++ = '\0';
- regcode = ptr;
-
- return(ret);
-}
-
-/*
- - regc - emit (if appropriate) a byte of code
- */
-static void
-regc( int b )
-{
- if (regcode != ®dummy)
- *regcode++ = b;
- else
- regsize++;
-}
-
-/*
- - reginsert - insert an operator in front of already-emitted operand
- *
- * Means relocating the operand.
- */
-static void
-reginsert(
- char op,
- char *opnd )
-{
- register char *src;
- register char *dst;
- register char *place;
-
- if (regcode == ®dummy) {
- regsize += 3;
- return;
- }
-
- src = regcode;
- regcode += 3;
- dst = regcode;
- while (src > opnd)
- *--dst = *--src;
-
- place = opnd; /* Op node, where operand used to be. */
- *place++ = op;
- *place++ = '\0';
- *place++ = '\0';
-}
-
-/*
- - regtail - set the next-pointer at the end of a node chain
- */
-static void
-regtail(
- char *p,
- char *val )
-{
- register char *scan;
- register char *temp;
- register int offset;
-
- if (p == ®dummy)
- return;
-
- /* Find last node. */
- scan = p;
- for (;;) {
- temp = regnext(scan);
- if (temp == NULL)
- break;
- scan = temp;
- }
-
- if (OP(scan) == BACK)
- offset = scan - val;
- else
- offset = val - scan;
- *(scan+1) = (offset>>8)&0377;
- *(scan+2) = offset&0377;
-}
-
-/*
- - regoptail - regtail on operand of first argument; nop if operandless
- */
-
-static void
-regoptail(
- char *p,
- char *val )
-{
- /* "Operandless" and "op != BRANCH" are synonymous in practice. */
- if (p == NULL || p == ®dummy || OP(p) != BRANCH)
- return;
- regtail(OPERAND(p), val);
-}
-
-/*
- * regexec and friends
- */
-
-/*
- * Global work variables for regexec().
- */
-static const char *reginput; /* String-input pointer. */
-static const char *regbol; /* Beginning of input, for ^ check. */
-static const char **regstartp; /* Pointer to startp array. */
-static const char **regendp; /* Ditto for endp. */
-
-/*
- * Forwards.
- */
-STATIC int regtry( regexp *prog, const char *string );
-STATIC int regmatch( char *prog );
-STATIC int regrepeat( char *p );
-
-#ifdef DEBUG
-int regnarrate = 0;
-void regdump();
-STATIC char *regprop();
-#endif
-
-/*
- - regexec - match a regexp against a string
- */
-int
-regexec(
- register regexp *prog,
- register const char *string )
-{
- register char *s;
-
- /* Be paranoid... */
- if (prog == NULL || string == NULL) {
- regerror("NULL parameter");
- return(0);
- }
-
- /* Check validity of program. */
- if (UCHARAT(prog->program) != MAGIC) {
- regerror("corrupted program");
- return(0);
- }
-
- /* If there is a "must appear" string, look for it. */
- if ( prog->regmust != NULL )
- {
- s = (char *)string;
- while ( ( s = strchr( s, prog->regmust[ 0 ] ) ) != NULL )
- {
- if ( !strncmp( s, prog->regmust, prog->regmlen ) )
- break; /* Found it. */
- ++s;
- }
- if ( s == NULL ) /* Not present. */
- return 0;
- }
-
- /* Mark beginning of line for ^ . */
- regbol = (char *)string;
-
- /* Simplest case: anchored match need be tried only once. */
- if ( prog->reganch )
- return regtry( prog, string );
-
- /* Messy cases: unanchored match. */
- s = (char *)string;
- if (prog->regstart != '\0')
- /* We know what char it must start with. */
- while ((s = strchr(s, prog->regstart)) != NULL) {
- if (regtry(prog, s))
- return(1);
- s++;
- }
- else
- /* We do not -- general case. */
- do {
- if ( regtry( prog, s ) )
- return( 1 );
- } while ( *s++ != '\0' );
-
- /* Failure. */
- return 0;
-}
-
-
-/*
- * regtry() - try match at specific point.
- */
-
-static int /* 0 failure, 1 success */
-regtry(
- regexp *prog,
- const char *string )
-{
- register int i;
- register const char * * sp;
- register const char * * ep;
-
- reginput = string;
- regstartp = prog->startp;
- regendp = prog->endp;
-
- sp = prog->startp;
- ep = prog->endp;
- for ( i = NSUBEXP; i > 0; --i )
- {
- *sp++ = NULL;
- *ep++ = NULL;
- }
- if ( regmatch( prog->program + 1 ) )
- {
- prog->startp[ 0 ] = string;
- prog->endp[ 0 ] = reginput;
- return 1;
- }
- else
- return 0;
-}
-
-
-/*
- * regmatch() - main matching routine.
- *
- * Conceptually the strategy is simple: check to see whether the current node
- * matches, call self recursively to see whether the rest matches, and then act
- * accordingly. In practice we make some effort to avoid recursion, in
- * particular by going through "ordinary" nodes (that do not need to know
- * whether the rest of the match failed) by a loop instead of by recursion.
- */
-
-static int /* 0 failure, 1 success */
-regmatch( char * prog )
-{
- char * scan; /* Current node. */
- char * next; /* Next node. */
-
- scan = prog;
-#ifdef DEBUG
- if (scan != NULL && regnarrate)
- fprintf(stderr, "%s(\n", regprop(scan));
-#endif
- while (scan != NULL) {
-#ifdef DEBUG
- if (regnarrate)
- fprintf(stderr, "%s...\n", regprop(scan));
-#endif
- next = regnext(scan);
-
- switch (OP(scan)) {
- case BOL:
- if (reginput != regbol)
- return(0);
- break;
- case EOL:
- if (*reginput != '\0')
- return(0);
- break;
- case WORDA:
- /* Must be looking at a letter, digit, or _ */
- if ((!isalnum(*reginput)) && *reginput != '_')
- return(0);
- /* Prev must be BOL or nonword */
- if (reginput > regbol &&
- (isalnum(reginput[-1]) || reginput[-1] == '_'))
- return(0);
- break;
- case WORDZ:
- /* Must be looking at non letter, digit, or _ */
- if (isalnum(*reginput) || *reginput == '_')
- return(0);
- /* We don't care what the previous char was */
- break;
- case ANY:
- if (*reginput == '\0')
- return(0);
- reginput++;
- break;
- case EXACTLY: {
- register int len;
- register char *opnd;
-
- opnd = OPERAND(scan);
- /* Inline the first character, for speed. */
- if (*opnd != *reginput)
- return(0);
- len = strlen(opnd);
- if (len > 1 && strncmp(opnd, reginput, len) != 0)
- return(0);
- reginput += len;
- }
- break;
- case ANYOF:
- if (*reginput == '\0' || strchr(OPERAND(scan), *reginput) == NULL)
- return(0);
- reginput++;
- break;
- case ANYBUT:
- if (*reginput == '\0' || strchr(OPERAND(scan), *reginput) != NULL)
- return(0);
- reginput++;
- break;
- case NOTHING:
- break;
- case BACK:
- break;
- case OPEN+1:
- case OPEN+2:
- case OPEN+3:
- case OPEN+4:
- case OPEN+5:
- case OPEN+6:
- case OPEN+7:
- case OPEN+8:
- case OPEN+9: {
- register int no;
- register const char *save;
-
- no = OP(scan) - OPEN;
- save = reginput;
-
- if (regmatch(next)) {
- /*
- * Don't set startp if some later
- * invocation of the same parentheses
- * already has.
- */
- if (regstartp[no] == NULL)
- regstartp[no] = save;
- return(1);
- } else
- return(0);
- }
- break;
- case CLOSE+1:
- case CLOSE+2:
- case CLOSE+3:
- case CLOSE+4:
- case CLOSE+5:
- case CLOSE+6:
- case CLOSE+7:
- case CLOSE+8:
- case CLOSE+9: {
- register int no;
- register const char *save;
-
- no = OP(scan) - CLOSE;
- save = reginput;
-
- if (regmatch(next)) {
- /*
- * Don't set endp if some later
- * invocation of the same parentheses
- * already has.
- */
- if (regendp[no] == NULL)
- regendp[no] = save;
- return(1);
- } else
- return(0);
- }
- break;
- case BRANCH: {
- register const char *save;
-
- if (OP(next) != BRANCH) /* No choice. */
- next = OPERAND(scan); /* Avoid recursion. */
- else {
- do {
- save = reginput;
- if (regmatch(OPERAND(scan)))
- return(1);
- reginput = save;
- scan = regnext(scan);
- } while (scan != NULL && OP(scan) == BRANCH);
- return(0);
- /* NOTREACHED */
- }
- }
- break;
- case STAR:
- case PLUS: {
- register char nextch;
- register int no;
- register const char *save;
- register int min;
-
- /*
- * Lookahead to avoid useless match attempts
- * when we know what character comes next.
- */
- nextch = '\0';
- if (OP(next) == EXACTLY)
- nextch = *OPERAND(next);
- min = (OP(scan) == STAR) ? 0 : 1;
- save = reginput;
- no = regrepeat(OPERAND(scan));
- while (no >= min) {
- /* If it could work, try it. */
- if (nextch == '\0' || *reginput == nextch)
- if (regmatch(next))
- return(1);
- /* Couldn't or didn't -- back up. */
- no--;
- reginput = save + no;
- }
- return(0);
- }
- break;
- case END:
- return(1); /* Success! */
- break;
- default:
- regerror("memory corruption");
- return(0);
- break;
- }
-
- scan = next;
- }
-
- /*
- * We get here only if there's trouble -- normally "case END" is
- * the terminating point.
- */
- regerror("corrupted pointers");
- return(0);
-}
-
-/*
- - regrepeat - repeatedly match something simple, report how many
- */
-static int
-regrepeat( char *p )
-{
- register int count = 0;
- register const char *scan;
- register char *opnd;
-
- scan = reginput;
- opnd = OPERAND(p);
- switch (OP(p)) {
- case ANY:
- count = strlen(scan);
- scan += count;
- break;
- case EXACTLY:
- while (*opnd == *scan) {
- count++;
- scan++;
- }
- break;
- case ANYOF:
- while (*scan != '\0' && strchr(opnd, *scan) != NULL) {
- count++;
- scan++;
- }
- break;
- case ANYBUT:
- while (*scan != '\0' && strchr(opnd, *scan) == NULL) {
- count++;
- scan++;
- }
- break;
- default: /* Oh dear. Called inappropriately. */
- regerror("internal foulup");
- count = 0; /* Best compromise. */
- break;
- }
- reginput = scan;
-
- return(count);
-}
-
-/*
- - regnext - dig the "next" pointer out of a node
- */
-static char *
-regnext( register char *p )
-{
- register int offset;
-
- if (p == ®dummy)
- return(NULL);
-
- offset = NEXT(p);
- if (offset == 0)
- return(NULL);
-
- if (OP(p) == BACK)
- return(p-offset);
- else
- return(p+offset);
-}
-
-#ifdef DEBUG
-
-STATIC char *regprop();
-
-/*
- - regdump - dump a regexp onto stdout in vaguely comprehensible form
- */
-void
-regdump( regexp *r )
-{
- register char *s;
- register char op = EXACTLY; /* Arbitrary non-END op. */
- register char *next;
-
-
- s = r->program + 1;
- while (op != END) { /* While that wasn't END last time... */
- op = OP(s);
- printf("%2d%s", s-r->program, regprop(s)); /* Where, what. */
- next = regnext(s);
- if (next == NULL) /* Next ptr. */
- printf("(0)");
- else
- printf("(%d)", (s-r->program)+(next-s));
- s += 3;
- if (op == ANYOF || op == ANYBUT || op == EXACTLY) {
- /* Literal string, where present. */
- while (*s != '\0') {
- putchar(*s);
- s++;
- }
- s++;
- }
- putchar('\n');
- }
-
- /* Header fields of interest. */
- if (r->regstart != '\0')
- printf("start `%c' ", r->regstart);
- if (r->reganch)
- printf("anchored ");
- if (r->regmust != NULL)
- printf("must have \"%s\"", r->regmust);
- printf("\n");
-}
-
-/*
- - regprop - printable representation of opcode
- */
-static char *
-regprop( char *op )
-{
- register char *p;
- static char buf[50];
-
- (void) strcpy(buf, ":");
-
- switch (OP(op)) {
- case BOL:
- p = "BOL";
- break;
- case EOL:
- p = "EOL";
- break;
- case ANY:
- p = "ANY";
- break;
- case ANYOF:
- p = "ANYOF";
- break;
- case ANYBUT:
- p = "ANYBUT";
- break;
- case BRANCH:
- p = "BRANCH";
- break;
- case EXACTLY:
- p = "EXACTLY";
- break;
- case NOTHING:
- p = "NOTHING";
- break;
- case BACK:
- p = "BACK";
- break;
- case END:
- p = "END";
- break;
- case OPEN+1:
- case OPEN+2:
- case OPEN+3:
- case OPEN+4:
- case OPEN+5:
- case OPEN+6:
- case OPEN+7:
- case OPEN+8:
- case OPEN+9:
- sprintf(buf+strlen(buf), "OPEN%d", OP(op)-OPEN);
- p = NULL;
- break;
- case CLOSE+1:
- case CLOSE+2:
- case CLOSE+3:
- case CLOSE+4:
- case CLOSE+5:
- case CLOSE+6:
- case CLOSE+7:
- case CLOSE+8:
- case CLOSE+9:
- sprintf(buf+strlen(buf), "CLOSE%d", OP(op)-CLOSE);
- p = NULL;
- break;
- case STAR:
- p = "STAR";
- break;
- case PLUS:
- p = "PLUS";
- break;
- case WORDA:
- p = "WORDA";
- break;
- case WORDZ:
- p = "WORDZ";
- break;
- default:
- regerror("corrupted opcode");
- break;
- }
- if (p != NULL)
- (void) strcat(buf, p);
- return(buf);
-}
-#endif
-
-/*
- * The following is provided for those people who do not have strcspn() in
- * their C libraries. They should get off their butts and do something
- * about it; at least one public-domain implementation of those (highly
- * useful) string routines has been published on Usenet.
- */
-#ifdef STRCSPN
-/*
- * strcspn - find length of initial segment of s1 consisting entirely
- * of characters not from s2
- */
-
-static int
-strcspn(
- char *s1,
- char *s2 )
-{
- register char *scan1;
- register char *scan2;
- register int count;
-
- count = 0;
- for (scan1 = s1; *scan1 != '\0'; scan1++) {
- for (scan2 = s2; *scan2 != '\0';) /* ++ moved down. */
- if (*scan1 == *scan2++)
- return(count);
- count++;
- }
- return(count);
-}
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/regexp.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/regexp.h b/ext/kenlm/jam-files/engine/regexp.h
deleted file mode 100644
index 6898ccd..0000000
--- a/ext/kenlm/jam-files/engine/regexp.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Definitions etc. for regexp(3) routines.
- *
- * Caveat: this is V8 regexp(3) [actually, a reimplementation thereof],
- * not the System V one.
- */
-#ifndef REGEXP_DWA20011023_H
-#define REGEXP_DWA20011023_H
-
-#define NSUBEXP 10
-typedef struct regexp {
- char const * startp[ NSUBEXP ];
- char const * endp[ NSUBEXP ];
- char regstart; /* Internal use only. */
- char reganch; /* Internal use only. */
- char * regmust; /* Internal use only. */
- int regmlen; /* Internal use only. */
- char program[ 1 ]; /* Unwarranted chumminess with compiler. */
-} regexp;
-
-
-regexp * regcomp( char const * exp );
-int regexec( regexp * prog, char const * string );
-void regerror( char const * s );
-
-
-/*
- * The first byte of the regexp internal "program" is actually this magic
- * number; the start node begins in the second byte.
- */
-#define MAGIC 0234
-
-#endif
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/rules.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/rules.c b/ext/kenlm/jam-files/engine/rules.c
deleted file mode 100644
index 7947c55..0000000
--- a/ext/kenlm/jam-files/engine/rules.c
+++ /dev/null
@@ -1,740 +0,0 @@
-/*
- * Copyright 1993, 1995 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * rules.c - access to RULEs, TARGETs, and ACTIONs
- *
- * External routines:
- * bindrule() - return pointer to RULE, creating it if necessary.
- * bindtarget() - return pointer to TARGET, creating it if necessary.
- * touch_target() - mark a target to simulate being new.
- * targetlist() - turn list of target names into a TARGET chain.
- * targetentry() - add a TARGET to a chain of TARGETS.
- * actionlist() - append to an ACTION chain.
- * addsettings() - add a deferred "set" command to a target.
- * pushsettings() - set all target specific variables.
- * popsettings() - reset target specific variables to their pre-push values.
- * freesettings() - delete a settings list.
- * rules_done() - free RULE and TARGET tables.
- */
-
-#include "jam.h"
-#include "rules.h"
-
-#include "hash.h"
-#include "lists.h"
-#include "object.h"
-#include "parse.h"
-#include "pathsys.h"
-#include "search.h"
-#include "variable.h"
-
-
-static void set_rule_actions( RULE *, rule_actions * );
-static void set_rule_body ( RULE *, FUNCTION * );
-
-static struct hash * targethash = 0;
-
-
-/*
- * get_target_includes() - lazy creates a target's internal includes node
- *
- * The newly created node is not entered into the hash table as there should
- * never be a need to bind them directly from a target names. If you want to
- * access an internal includes node by name, first access the actual target and
- * then read the internal includes node from there.
- */
-
-static TARGET * get_target_includes( TARGET * const t )
-{
- if ( !t->includes )
- {
- TARGET * const i = (TARGET *)BJAM_MALLOC( sizeof( *t ) );
- memset( (char *)i, '\0', sizeof( *i ) );
- i->name = object_copy( t->name );
- i->boundname = object_copy( i->name );
- i->flags |= T_FLAG_NOTFILE | T_FLAG_INTERNAL;
- i->original_target = t;
- t->includes = i;
- }
- return t->includes;
-}
-
-
-/*
- * target_include() - adds a target to the given targe's 'included' list
- * target_include_many() - adds targets to the given target's 'included' list
- *
- * Included targets are modeled as dependencies of the including target's
- * internal include node.
- */
-
-void target_include( TARGET * const including, TARGET * const included )
-{
- TARGET * const internal = get_target_includes( including );
- internal->depends = targetentry( internal->depends, included );
-}
-
-void target_include_many( TARGET * const including, LIST * const included_names
- )
-{
- TARGET * const internal = get_target_includes( including );
- internal->depends = targetlist( internal->depends, included_names );
-}
-
-
-/*
- * enter_rule() - return pointer to RULE, creating it if necessary in
- * target_module.
- */
-
-static RULE * enter_rule( OBJECT * rulename, module_t * target_module )
-{
- int found;
- RULE * const r = (RULE *)hash_insert( demand_rules( target_module ),
- rulename, &found );
- if ( !found )
- {
- r->name = object_copy( rulename );
- r->procedure = 0;
- r->module = 0;
- r->actions = 0;
- r->exported = 0;
- r->module = target_module;
- }
- return r;
-}
-
-
-/*
- * define_rule() - return pointer to RULE, creating it if necessary in
- * target_module. Prepare it to accept a body or action originating in
- * src_module.
- */
-
-static RULE * define_rule( module_t * src_module, OBJECT * rulename,
- module_t * target_module )
-{
- RULE * const r = enter_rule( rulename, target_module );
- if ( r->module != src_module )
- {
- /* If the rule was imported from elsewhere, clear it now. */
- set_rule_body( r, 0 );
- set_rule_actions( r, 0 );
- /* r will be executed in the source module. */
- r->module = src_module;
- }
- return r;
-}
-
-
-void rule_free( RULE * r )
-{
- object_free( r->name );
- r->name = 0;
- if ( r->procedure )
- function_free( r->procedure );
- r->procedure = 0;
- if ( r->actions )
- actions_free( r->actions );
- r->actions = 0;
-}
-
-
-/*
- * bindtarget() - return pointer to TARGET, creating it if necessary.
- */
-
-TARGET * bindtarget( OBJECT * const target_name )
-{
- int found;
- TARGET * t;
-
- if ( !targethash )
- targethash = hashinit( sizeof( TARGET ), "targets" );
-
- t = (TARGET *)hash_insert( targethash, target_name, &found );
- if ( !found )
- {
- memset( (char *)t, '\0', sizeof( *t ) );
- t->name = object_copy( target_name );
- t->boundname = object_copy( t->name ); /* default for T_FLAG_NOTFILE */
- }
-
- return t;
-}
-
-
-static void bind_explicitly_located_target( void * xtarget, void * data )
-{
- TARGET * t = (TARGET *)xtarget;
- if ( !( t->flags & T_FLAG_NOTFILE ) )
- {
- /* Check if there is a setting for LOCATE. */
- SETTINGS * s = t->settings;
- for ( ; s ; s = s->next )
- {
- if ( object_equal( s->symbol, constant_LOCATE ) && ! list_empty( s->value ) )
- {
- set_explicit_binding( t->name, list_front( s->value ) );
- break;
- }
- }
- }
-}
-
-
-void bind_explicitly_located_targets()
-{
- if ( targethash )
- hashenumerate( targethash, bind_explicitly_located_target, (void *)0 );
-}
-
-
-/*
- * touch_target() - mark a target to simulate being new.
- */
-
-void touch_target( OBJECT * const t )
-{
- bindtarget( t )->flags |= T_FLAG_TOUCHED;
-}
-
-
-/*
- * target_scc() - returns the root of a strongly connected component that this
- * target is a part of.
- */
-
-TARGET * target_scc( TARGET * t )
-{
- TARGET * result = t;
- while ( result->scc_root )
- result = result->scc_root;
- while ( t->scc_root )
- {
- TARGET * const tmp = t->scc_root;
- t->scc_root = result;
- t = tmp;
- }
- return result;
-}
-
-
-/*
- * targetlist() - turn list of target names into a TARGET chain.
- *
- * Inputs:
- * chain existing TARGETS to append to
- * targets list of target names
- */
-
-TARGETS * targetlist( TARGETS * chain, LIST * target_names )
-{
- LISTITER iter = list_begin( target_names );
- LISTITER const end = list_end( target_names );
- for ( ; iter != end; iter = list_next( iter ) )
- chain = targetentry( chain, bindtarget( list_item( iter ) ) );
- return chain;
-}
-
-
-/*
- * targetentry() - add a TARGET to a chain of TARGETS.
- *
- * Inputs:
- * chain existing TARGETS to append to
- * target new target to append
- */
-
-TARGETS * targetentry( TARGETS * chain, TARGET * target )
-{
- TARGETS * const c = (TARGETS *)BJAM_MALLOC( sizeof( TARGETS ) );
- c->target = target;
-
- if ( !chain ) chain = c;
- else chain->tail->next = c;
- chain->tail = c;
- c->next = 0;
-
- return chain;
-}
-
-
-/*
- * targetchain() - append two TARGET chains.
- *
- * Inputs:
- * chain existing TARGETS to append to
- * target new target to append
- */
-
-TARGETS * targetchain( TARGETS * chain, TARGETS * targets )
-{
- if ( !targets ) return chain;
- if ( !chain ) return targets;
-
- chain->tail->next = targets;
- chain->tail = targets->tail;
- return chain;
-}
-
-/*
- * action_free - decrement the ACTIONs refrence count and (maybe) free it.
- */
-
-void action_free( ACTION * action )
-{
- if ( --action->refs == 0 )
- {
- freetargets( action->targets );
- freetargets( action->sources );
- BJAM_FREE( action );
- }
-}
-
-
-/*
- * actionlist() - append to an ACTION chain.
- */
-
-ACTIONS * actionlist( ACTIONS * chain, ACTION * action )
-{
- ACTIONS * const actions = (ACTIONS *)BJAM_MALLOC( sizeof( ACTIONS ) );
- actions->action = action;
- ++action->refs;
- if ( !chain ) chain = actions;
- else chain->tail->next = actions;
- chain->tail = actions;
- actions->next = 0;
- return chain;
-}
-
-static SETTINGS * settings_freelist;
-
-
-/*
- * addsettings() - add a deferred "set" command to a target.
- *
- * Adds a variable setting (varname=list) onto a chain of settings for a
- * particular target. 'flag' controls the relationship between new and old
- * values in the same way as in var_set() function (see variable.c). Returns the
- * head of the settings chain.
- */
-
-SETTINGS * addsettings( SETTINGS * head, int flag, OBJECT * symbol,
- LIST * value )
-{
- SETTINGS * v;
-
- /* Look for previous settings. */
- for ( v = head; v; v = v->next )
- if ( object_equal( v->symbol, symbol ) )
- break;
-
- /* If not previously set, alloc a new. */
- /* If appending, do so. */
- /* Else free old and set new. */
- if ( !v )
- {
- v = settings_freelist;
- if ( v )
- settings_freelist = v->next;
- else
- v = (SETTINGS *)BJAM_MALLOC( sizeof( *v ) );
-
- v->symbol = object_copy( symbol );
- v->value = value;
- v->next = head;
- head = v;
- }
- else if ( flag == VAR_APPEND )
- {
- v->value = list_append( v->value, value );
- }
- else if ( flag != VAR_DEFAULT )
- {
- list_free( v->value );
- v->value = value;
- }
- else
- list_free( value );
-
- /* Return (new) head of list. */
- return head;
-}
-
-
-/*
- * pushsettings() - set all target specific variables.
- */
-
-void pushsettings( struct module_t * module, SETTINGS * v )
-{
- for ( ; v; v = v->next )
- v->value = var_swap( module, v->symbol, v->value );
-}
-
-
-/*
- * popsettings() - reset target specific variables to their pre-push values.
- */
-
-void popsettings( struct module_t * module, SETTINGS * v )
-{
- pushsettings( module, v ); /* just swap again */
-}
-
-
-/*
- * copysettings() - duplicate a settings list, returning the new copy.
- */
-
-SETTINGS * copysettings( SETTINGS * head )
-{
- SETTINGS * copy = 0;
- SETTINGS * v;
- for ( v = head; v; v = v->next )
- copy = addsettings( copy, VAR_SET, v->symbol, list_copy( v->value ) );
- return copy;
-}
-
-
-/*
- * freetargets() - delete a targets list.
- */
-
-void freetargets( TARGETS * chain )
-{
- while ( chain )
- {
- TARGETS * const n = chain->next;
- BJAM_FREE( chain );
- chain = n;
- }
-}
-
-
-/*
- * freeactions() - delete an action list.
- */
-
-void freeactions( ACTIONS * chain )
-{
- while ( chain )
- {
- ACTIONS * const n = chain->next;
- action_free( chain->action );
- BJAM_FREE( chain );
- chain = n;
- }
-}
-
-
-/*
- * freesettings() - delete a settings list.
- */
-
-void freesettings( SETTINGS * v )
-{
- while ( v )
- {
- SETTINGS * const n = v->next;
- object_free( v->symbol );
- list_free( v->value );
- v->next = settings_freelist;
- settings_freelist = v;
- v = n;
- }
-}
-
-
-static void freetarget( void * xt, void * data )
-{
- TARGET * const t = (TARGET *)xt;
- if ( t->name ) object_free ( t->name );
- if ( t->boundname ) object_free ( t->boundname );
- if ( t->settings ) freesettings( t->settings );
- if ( t->depends ) freetargets ( t->depends );
- if ( t->dependants ) freetargets ( t->dependants );
- if ( t->parents ) freetargets ( t->parents );
- if ( t->actions ) freeactions ( t->actions );
- if ( t->includes )
- {
- freetarget( t->includes, (void *)0 );
- BJAM_FREE( t->includes );
- }
-}
-
-
-/*
- * rules_done() - free RULE and TARGET tables.
- */
-
-void rules_done()
-{
- if ( targethash )
- {
- hashenumerate( targethash, freetarget, 0 );
- hashdone( targethash );
- }
- while ( settings_freelist )
- {
- SETTINGS * const n = settings_freelist->next;
- BJAM_FREE( settings_freelist );
- settings_freelist = n;
- }
-}
-
-
-/*
- * actions_refer() - add a new reference to the given actions.
- */
-
-void actions_refer( rule_actions * a )
-{
- ++a->reference_count;
-}
-
-
-/*
- * actions_free() - release a reference to given actions.
- */
-
-void actions_free( rule_actions * a )
-{
- if ( --a->reference_count <= 0 )
- {
- function_free( a->command );
- list_free( a->bindlist );
- BJAM_FREE( a );
- }
-}
-
-
-/*
- * set_rule_body() - set the argument list and procedure of the given rule.
- */
-
-static void set_rule_body( RULE * rule, FUNCTION * procedure )
-{
- if ( procedure )
- function_refer( procedure );
- if ( rule->procedure )
- function_free( rule->procedure );
- rule->procedure = procedure;
-}
-
-
-/*
- * global_name() - given a rule, return the name for a corresponding rule in the
- * global module.
- */
-
-static OBJECT * global_rule_name( RULE * r )
-{
- if ( r->module == root_module() )
- return object_copy( r->name );
-
- {
- char name[ 4096 ] = "";
- if ( r->module->name )
- {
- strncat( name, object_str( r->module->name ), sizeof( name ) - 1 );
- strncat( name, ".", sizeof( name ) - 1 );
- }
- strncat( name, object_str( r->name ), sizeof( name ) - 1 );
- return object_new( name );
- }
-}
-
-
-/*
- * global_rule() - given a rule, produce a corresponding entry in the global
- * module.
- */
-
-static RULE * global_rule( RULE * r )
-{
- if ( r->module == root_module() )
- return r;
-
- {
- OBJECT * const name = global_rule_name( r );
- RULE * const result = define_rule( r->module, name, root_module() );
- object_free( name );
- return result;
- }
-}
-
-
-/*
- * new_rule_body() - make a new rule named rulename in the given module, with
- * the given argument list and procedure. If exported is true, the rule is
- * exported to the global module as modulename.rulename.
- */
-
-RULE * new_rule_body( module_t * m, OBJECT * rulename, FUNCTION * procedure,
- int exported )
-{
- RULE * const local = define_rule( m, rulename, m );
- local->exported = exported;
- set_rule_body( local, procedure );
-
- /* Mark the procedure with the global rule name, regardless of whether the
- * rule is exported. That gives us something reasonably identifiable that we
- * can use, e.g. in profiling output. Only do this once, since this could be
- * called multiple times with the same procedure.
- */
- if ( !function_rulename( procedure ) )
- function_set_rulename( procedure, global_rule_name( local ) );
-
- return local;
-}
-
-
-static void set_rule_actions( RULE * rule, rule_actions * actions )
-{
- if ( actions )
- actions_refer( actions );
- if ( rule->actions )
- actions_free( rule->actions );
- rule->actions = actions;
-}
-
-
-static rule_actions * actions_new( FUNCTION * command, LIST * bindlist,
- int flags )
-{
- rule_actions * const result = (rule_actions *)BJAM_MALLOC( sizeof(
- rule_actions ) );
- function_refer( command );
- result->command = command;
- result->bindlist = bindlist;
- result->flags = flags;
- result->reference_count = 0;
- return result;
-}
-
-
-RULE * new_rule_actions( module_t * m, OBJECT * rulename, FUNCTION * command,
- LIST * bindlist, int flags )
-{
- RULE * const local = define_rule( m, rulename, m );
- RULE * const global = global_rule( local );
- set_rule_actions( local, actions_new( command, bindlist, flags ) );
- set_rule_actions( global, local->actions );
- return local;
-}
-
-
-/*
- * Looks for a rule in the specified module, and returns it, if found. First
- * checks if the rule is present in the module's rule table. Second, if the
- * rule's name is in the form name1.name2 and name1 is in the list of imported
- * modules, look in module 'name1' for rule 'name2'.
- */
-
-RULE * lookup_rule( OBJECT * rulename, module_t * m, int local_only )
-{
- RULE * r;
- RULE * result = 0;
- module_t * original_module = m;
-
- if ( m->class_module )
- m = m->class_module;
-
- if ( m->rules && ( r = (RULE *)hash_find( m->rules, rulename ) ) )
- result = r;
- else if ( !local_only && m->imported_modules )
- {
- /* Try splitting the name into module and rule. */
- char * p = strchr( object_str( rulename ), '.' ) ;
- if ( p )
- {
- /* Now, r->name keeps the module name, and p + 1 keeps the rule
- * name.
- */
- OBJECT * rule_part = object_new( p + 1 );
- OBJECT * module_part;
- {
- string buf[ 1 ];
- string_new( buf );
- string_append_range( buf, object_str( rulename ), p );
- module_part = object_new( buf->value );
- string_free( buf );
- }
- if ( hash_find( m->imported_modules, module_part ) )
- result = lookup_rule( rule_part, bindmodule( module_part ), 1 );
- object_free( module_part );
- object_free( rule_part );
- }
- }
-
- if ( result )
- {
- if ( local_only && !result->exported )
- result = 0;
- else if ( original_module != m )
- {
- /* Lookup started in class module. We have found a rule in class
- * module, which is marked for execution in that module, or in some
- * instance. Mark it for execution in the instance where we started
- * the lookup.
- */
- int const execute_in_class = result->module == m;
- int const execute_in_some_instance =
- result->module->class_module == m;
- if ( execute_in_class || execute_in_some_instance )
- result->module = original_module;
- }
- }
-
- return result;
-}
-
-
-RULE * bindrule( OBJECT * rulename, module_t * m )
-{
- RULE * result = lookup_rule( rulename, m, 0 );
- if ( !result )
- result = lookup_rule( rulename, root_module(), 0 );
- /* We have only one caller, 'evaluate_rule', which will complain about
- * calling an undefined rule. We could issue the error here, but we do not
- * have the necessary information, such as frame.
- */
- if ( !result )
- result = enter_rule( rulename, m );
- return result;
-}
-
-
-RULE * import_rule( RULE * source, module_t * m, OBJECT * name )
-{
- RULE * const dest = define_rule( source->module, name, m );
- set_rule_body( dest, source->procedure );
- set_rule_actions( dest, source->actions );
- return dest;
-}
-
-
-void rule_localize( RULE * rule, module_t * m )
-{
- rule->module = m;
- if ( rule->procedure )
- {
- FUNCTION * procedure = function_unbind_variables( rule->procedure );
- function_refer( procedure );
- function_free( rule->procedure );
- rule->procedure = procedure;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/rules.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/rules.h b/ext/kenlm/jam-files/engine/rules.h
deleted file mode 100644
index fe2792f..0000000
--- a/ext/kenlm/jam-files/engine/rules.h
+++ /dev/null
@@ -1,270 +0,0 @@
-/*
- * Copyright 1993, 1995 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * rules.h - targets, rules, and related information
- *
- * This file describes the structures holding the targets, rules, and related
- * information accumulated by interpreting the statements of the jam files.
- *
- * The following are defined:
- *
- * RULE - a generic jam rule, the product of RULE and ACTIONS.
- * ACTIONS - a chain of ACTIONs.
- * ACTION - a RULE instance with targets and sources.
- * SETTINGS - variables to set when executing a TARGET's ACTIONS.
- * TARGETS - a chain of TARGETs.
- * TARGET - an entity (e.g. a file) that can be built.
- */
-
-#ifndef RULES_DWA_20011020_H
-#define RULES_DWA_20011020_H
-
-#include "function.h"
-#include "modules.h"
-#include "timestamp.h"
-
-
-typedef struct _rule RULE;
-typedef struct _target TARGET;
-typedef struct _targets TARGETS;
-typedef struct _action ACTION;
-typedef struct _actions ACTIONS;
-typedef struct _settings SETTINGS ;
-
-/* RULE - a generic jam rule, the product of RULE and ACTIONS. */
-
-/* Build actions corresponding to a rule. */
-struct rule_actions
-{
- int reference_count;
- FUNCTION * command; /* command string from ACTIONS */
- LIST * bindlist;
- int flags; /* modifiers on ACTIONS */
-
-#define RULE_NEWSRCS 0x01 /* $(>) is updated sources only */
-#define RULE_TOGETHER 0x02 /* combine actions on single target */
-#define RULE_IGNORE 0x04 /* ignore return status of executes */
-#define RULE_QUIETLY 0x08 /* do not mention it unless verbose */
-#define RULE_PIECEMEAL 0x10 /* split exec so each $(>) is small */
-#define RULE_EXISTING 0x20 /* $(>) is pre-exisitng sources only */
-};
-
-typedef struct rule_actions rule_actions;
-typedef struct argument_list argument_list;
-
-struct _rule
-{
- OBJECT * name;
- FUNCTION * procedure;
- rule_actions * actions; /* build actions, or NULL for no actions */
- module_t * module; /* module in which this rule is executed */
- int exported; /* nonzero if this rule is supposed to appear in
- * the global module and be automatically
- * imported into other modules
- */
-};
-
-/* ACTIONS - a chain of ACTIONs. */
-struct _actions
-{
- ACTIONS * next;
- ACTIONS * tail; /* valid only for head */
- ACTION * action;
-};
-
-/* ACTION - a RULE instance with targets and sources. */
-struct _action
-{
- RULE * rule;
- TARGETS * targets;
- TARGETS * sources; /* aka $(>) */
- char running; /* has been started */
-#define A_INIT 0
-#define A_RUNNING_NOEXEC 1
-#define A_RUNNING 2
- char status; /* see TARGET status */
- int refs;
-};
-
-/* SETTINGS - variables to set when executing a TARGET's ACTIONS. */
-struct _settings
-{
- SETTINGS * next;
- OBJECT * symbol; /* symbol name for var_set() */
- LIST * value; /* symbol value for var_set() */
-};
-
-/* TARGETS - a chain of TARGETs. */
-struct _targets
-{
- TARGETS * next;
- TARGETS * tail; /* valid only for head */
- TARGET * target;
-};
-
-/* TARGET - an entity (e.g. a file) that can be built. */
-struct _target
-{
- OBJECT * name;
- OBJECT * boundname; /* if search() relocates target */
- ACTIONS * actions; /* rules to execute, if any */
- SETTINGS * settings; /* variables to define */
-
- short flags; /* status info */
-
-#define T_FLAG_TEMP 0x0001 /* TEMPORARY applied */
-#define T_FLAG_NOCARE 0x0002 /* NOCARE applied */
-#define T_FLAG_NOTFILE 0x0004 /* NOTFILE applied */
-#define T_FLAG_TOUCHED 0x0008 /* ALWAYS applied or -t target */
-#define T_FLAG_LEAVES 0x0010 /* LEAVES applied */
-#define T_FLAG_NOUPDATE 0x0020 /* NOUPDATE applied */
-#define T_FLAG_VISITED 0x0040 /* CWM: Used in debugging */
-
-/* This flag has been added to support a new built-in rule named "RMBAD". It is
- * used to force removal of outdated targets whose dependencies fail to build.
- */
-#define T_FLAG_RMOLD 0x0080 /* RMBAD applied */
-
-/* This flag was added to support a new built-in rule named "FAIL_EXPECTED" used
- * to indicate that the result of running a given action should be inverted,
- * i.e. ok <=> fail. Useful for launching certain test runs from a Jamfile.
- */
-#define T_FLAG_FAIL_EXPECTED 0x0100 /* FAIL_EXPECTED applied */
-
-#define T_FLAG_INTERNAL 0x0200 /* internal INCLUDES node */
-
-/* Indicates that the target must be a file. Prevents matching non-files, like
- * directories, when a target is searched.
- */
-#define T_FLAG_ISFILE 0x0400
-
-#define T_FLAG_PRECIOUS 0x0800
-
- char binding; /* how target relates to a real file or
- * folder
- */
-
-#define T_BIND_UNBOUND 0 /* a disembodied name */
-#define T_BIND_MISSING 1 /* could not find real file */
-#define T_BIND_PARENTS 2 /* using parent's timestamp */
-#define T_BIND_EXISTS 3 /* real file, timestamp valid */
-
- TARGETS * depends; /* dependencies */
- TARGETS * dependants; /* the inverse of dependencies */
- TARGETS * rebuilds; /* targets that should be force-rebuilt
- * whenever this one is
- */
- TARGET * includes; /* internal includes node */
- TARGET * original_target; /* original_target->includes = this */
- char rescanned;
-
- timestamp time; /* update time */
- timestamp leaf; /* update time of leaf sources */
-
- char fate; /* make0()'s diagnosis */
-
-#define T_FATE_INIT 0 /* nothing done to target */
-#define T_FATE_MAKING 1 /* make0(target) on stack */
-
-#define T_FATE_STABLE 2 /* target did not need updating */
-#define T_FATE_NEWER 3 /* target newer than parent */
-
-#define T_FATE_SPOIL 4 /* >= SPOIL rebuilds parents */
-#define T_FATE_ISTMP 4 /* unneeded temp target oddly present */
-
-#define T_FATE_BUILD 5 /* >= BUILD rebuilds target */
-#define T_FATE_TOUCHED 5 /* manually touched with -t */
-#define T_FATE_REBUILD 6
-#define T_FATE_MISSING 7 /* is missing, needs updating */
-#define T_FATE_NEEDTMP 8 /* missing temp that must be rebuild */
-#define T_FATE_OUTDATED 9 /* is out of date, needs updating */
-#define T_FATE_UPDATE 10 /* deps updated, needs updating */
-
-#define T_FATE_BROKEN 11 /* >= BROKEN ruins parents */
-#define T_FATE_CANTFIND 11 /* no rules to make missing target */
-#define T_FATE_CANTMAKE 12 /* can not find dependencies */
-
- char progress; /* tracks make1() progress */
-
-#define T_MAKE_INIT 0 /* make1(target) not yet called */
-#define T_MAKE_ONSTACK 1 /* make1(target) on stack */
-#define T_MAKE_ACTIVE 2 /* make1(target) in make1b() */
-#define T_MAKE_RUNNING 3 /* make1(target) running commands */
-#define T_MAKE_DONE 4 /* make1(target) done */
-#define T_MAKE_NOEXEC_DONE 5 /* make1(target) done with -n in effect */
-
-#ifdef OPT_SEMAPHORE
- #define T_MAKE_SEMAPHORE 5 /* Special target type for semaphores */
-#endif
-
-#ifdef OPT_SEMAPHORE
- TARGET * semaphore; /* used in serialization */
-#endif
-
- char status; /* exec_cmd() result */
-
- int asynccnt; /* child deps outstanding */
- TARGETS * parents; /* used by make1() for completion */
- TARGET * scc_root; /* used by make to resolve cyclic includes
- */
- TARGET * rescanning; /* used by make0 to mark visited targets
- * when rescanning
- */
- int depth; /* The depth of the target in the make0
- * stack.
- */
- char * cmds; /* type-punned command list */
-
- char const * failed;
-};
-
-
-/* Action related functions. */
-void action_free ( ACTION * );
-ACTIONS * actionlist ( ACTIONS *, ACTION * );
-void freeactions ( ACTIONS * );
-SETTINGS * addsettings ( SETTINGS *, int flag, OBJECT * symbol, LIST * value );
-void pushsettings ( module_t *, SETTINGS * );
-void popsettings ( module_t *, SETTINGS * );
-SETTINGS * copysettings ( SETTINGS * );
-void freesettings ( SETTINGS * );
-void actions_refer( rule_actions * );
-void actions_free ( rule_actions * );
-
-/* Rule related functions. */
-RULE * bindrule ( OBJECT * rulename, module_t * );
-RULE * import_rule ( RULE * source, module_t *, OBJECT * name );
-void rule_localize ( RULE * rule, module_t * module );
-RULE * new_rule_body ( module_t *, OBJECT * rulename, FUNCTION * func, int exprt );
-RULE * new_rule_actions( module_t *, OBJECT * rulename, FUNCTION * command, LIST * bindlist, int flags );
-void rule_free ( RULE * );
-
-/* Target related functions. */
-void bind_explicitly_located_targets();
-TARGET * bindtarget ( OBJECT * const );
-void freetargets ( TARGETS * );
-TARGETS * targetchain ( TARGETS *, TARGETS * );
-TARGETS * targetentry ( TARGETS *, TARGET * );
-void target_include ( TARGET * const including,
- TARGET * const included );
-void target_include_many ( TARGET * const including,
- LIST * const included_names );
-TARGETS * targetlist ( TARGETS *, LIST * target_names );
-void touch_target ( OBJECT * const );
-void clear_includes ( TARGET * );
-TARGET * target_scc ( TARGET * );
-
-/* Final module cleanup. */
-void rules_done();
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/scan.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/scan.c b/ext/kenlm/jam-files/engine/scan.c
deleted file mode 100644
index d92fdca..0000000
--- a/ext/kenlm/jam-files/engine/scan.c
+++ /dev/null
@@ -1,404 +0,0 @@
-/*
- * Copyright 1993-2002 Christopher Seiwald and Perforce Software, Inc.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/*
- * scan.c - the jam yacc scanner
- *
- */
-
-#include "jam.h"
-#include "scan.h"
-
-#include "constants.h"
-#include "jambase.h"
-#include "jamgram.h"
-
-
-struct keyword
-{
- char * word;
- int type;
-} keywords[] =
-{
-#include "jamgramtab.h"
- { 0, 0 }
-};
-
-typedef struct include include;
-struct include
-{
- include * next; /* next serial include file */
- char * string; /* pointer into current line */
- char * * strings; /* for yyfparse() -- text to parse */
- FILE * file; /* for yyfparse() -- file being read */
- OBJECT * fname; /* for yyfparse() -- file name */
- int line; /* line counter for error messages */
- char buf[ 512 ]; /* for yyfparse() -- line buffer */
-};
-
-static include * incp = 0; /* current file; head of chain */
-
-static int scanmode = SCAN_NORMAL;
-static int anyerrors = 0;
-
-
-static char * symdump( YYSTYPE * );
-
-#define BIGGEST_TOKEN 10240 /* no single token can be larger */
-
-
-/*
- * Set parser mode: normal, string, or keyword.
- */
-
-void yymode( int n )
-{
- scanmode = n;
-}
-
-
-void yyerror( char const * s )
-{
- /* We use yylval instead of incp to access the error location information as
- * the incp pointer will already be reset to 0 in case the error occurred at
- * EOF.
- *
- * The two may differ only if ran into an unexpected EOF or we get an error
- * while reading a lexical token spanning multiple lines, e.g. a multi-line
- * string literal or action body, in which case yylval location information
- * will hold the information about where the token started while incp will
- * hold the information about where reading it broke.
- */
- printf( "%s:%d: %s at %s\n", object_str( yylval.file ), yylval.line, s,
- symdump( &yylval ) );
- ++anyerrors;
-}
-
-
-int yyanyerrors()
-{
- return anyerrors != 0;
-}
-
-
-void yyfparse( OBJECT * s )
-{
- include * i = (include *)BJAM_MALLOC( sizeof( *i ) );
-
- /* Push this onto the incp chain. */
- i->string = "";
- i->strings = 0;
- i->file = 0;
- i->fname = object_copy( s );
- i->line = 0;
- i->next = incp;
- incp = i;
-
- /* If the filename is "+", it means use the internal jambase. */
- if ( !strcmp( object_str( s ), "+" ) )
- i->strings = jambase;
-}
-
-
-/*
- * yyline() - read new line and return first character.
- *
- * Fabricates a continuous stream of characters across include files, returning
- * EOF at the bitter end.
- */
-
-int yyline()
-{
- include * const i = incp;
-
- if ( !incp )
- return EOF;
-
- /* Once we start reading from the input stream, we reset the include
- * insertion point so that the next include file becomes the head of the
- * list.
- */
-
- /* If there is more data in this line, return it. */
- if ( *i->string )
- return *i->string++;
-
- /* If we are reading from an internal string list, go to the next string. */
- if ( i->strings )
- {
- if ( *i->strings )
- {
- ++i->line;
- i->string = *(i->strings++);
- return *i->string++;
- }
- }
- else
- {
- /* If necessary, open the file. */
- if ( !i->file )
- {
- FILE * f = stdin;
- if ( strcmp( object_str( i->fname ), "-" ) && !( f = fopen( object_str( i->fname ), "r" ) ) )
- perror( object_str( i->fname ) );
- i->file = f;
- }
-
- /* If there is another line in this file, start it. */
- if ( i->file && fgets( i->buf, sizeof( i->buf ), i->file ) )
- {
- ++i->line;
- i->string = i->buf;
- return *i->string++;
- }
- }
-
- /* This include is done. Free it up and return EOF so yyparse() returns to
- * parse_file().
- */
-
- incp = i->next;
-
- /* Close file, free name. */
- if ( i->file && ( i->file != stdin ) )
- fclose( i->file );
- object_free( i->fname );
- BJAM_FREE( (char *)i );
-
- return EOF;
-}
-
-
-/*
- * yylex() - set yylval to current token; return its type.
- *
- * Macros to move things along:
- *
- * yychar() - return and advance character; invalid after EOF.
- * yyprev() - back up one character; invalid before yychar().
- *
- * yychar() returns a continuous stream of characters, until it hits the EOF of
- * the current include file.
- */
-
-#define yychar() ( *incp->string ? *incp->string++ : yyline() )
-#define yyprev() ( incp->string-- )
-
-int yylex()
-{
- int c;
- char buf[ BIGGEST_TOKEN ];
- char * b = buf;
-
- if ( !incp )
- goto eof;
-
- /* Get first character (whitespace or of token). */
- c = yychar();
-
- if ( scanmode == SCAN_STRING )
- {
- /* If scanning for a string (action's {}'s), look for the closing brace.
- * We handle matching braces, if they match.
- */
-
- int nest = 1;
-
- while ( ( c != EOF ) && ( b < buf + sizeof( buf ) ) )
- {
- if ( c == '{' )
- ++nest;
-
- if ( ( c == '}' ) && !--nest )
- break;
-
- *b++ = c;
-
- c = yychar();
-
- /* Turn trailing "\r\n" sequences into plain "\n" for Cygwin. */
- if ( ( c == '\n' ) && ( b[ -1 ] == '\r' ) )
- --b;
- }
-
- /* We ate the ending brace -- regurgitate it. */
- if ( c != EOF )
- yyprev();
-
- /* Check for obvious errors. */
- if ( b == buf + sizeof( buf ) )
- {
- yyerror( "action block too big" );
- goto eof;
- }
-
- if ( nest )
- {
- yyerror( "unmatched {} in action block" );
- goto eof;
- }
-
- *b = 0;
- yylval.type = STRING;
- yylval.string = object_new( buf );
- yylval.file = incp->fname;
- yylval.line = incp->line;
- }
- else
- {
- char * b = buf;
- struct keyword * k;
- int inquote = 0;
- int notkeyword;
-
- /* Eat white space. */
- for ( ; ; )
- {
- /* Skip past white space. */
- while ( ( c != EOF ) && isspace( c ) )
- c = yychar();
-
- /* Not a comment? */
- if ( c != '#' )
- break;
-
- /* Swallow up comment line. */
- while ( ( ( c = yychar() ) != EOF ) && ( c != '\n' ) ) ;
- }
-
- /* c now points to the first character of a token. */
- if ( c == EOF )
- goto eof;
-
- yylval.file = incp->fname;
- yylval.line = incp->line;
-
- /* While scanning the word, disqualify it for (expensive) keyword lookup
- * when we can: $anything, "anything", \anything
- */
- notkeyword = c == '$';
-
- /* Look for white space to delimit word. "'s get stripped but preserve
- * white space. \ protects next character.
- */
- while
- (
- ( c != EOF ) &&
- ( b < buf + sizeof( buf ) ) &&
- ( inquote || !isspace( c ) )
- )
- {
- if ( c == '"' )
- {
- /* begin or end " */
- inquote = !inquote;
- notkeyword = 1;
- }
- else if ( c != '\\' )
- {
- /* normal char */
- *b++ = c;
- }
- else if ( ( c = yychar() ) != EOF )
- {
- /* \c */
- if (c == 'n')
- c = '\n';
- else if (c == 'r')
- c = '\r';
- else if (c == 't')
- c = '\t';
- *b++ = c;
- notkeyword = 1;
- }
- else
- {
- /* \EOF */
- break;
- }
-
- c = yychar();
- }
-
- /* Check obvious errors. */
- if ( b == buf + sizeof( buf ) )
- {
- yyerror( "string too big" );
- goto eof;
- }
-
- if ( inquote )
- {
- yyerror( "unmatched \" in string" );
- goto eof;
- }
-
- /* We looked ahead a character - back up. */
- if ( c != EOF )
- yyprev();
-
- /* Scan token table. Do not scan if it is obviously not a keyword or if
- * it is an alphabetic when were looking for punctuation.
- */
-
- *b = 0;
- yylval.type = ARG;
-
- if ( !notkeyword && !( isalpha( *buf ) && ( scanmode == SCAN_PUNCT ) ) )
- for ( k = keywords; k->word; ++k )
- if ( ( *buf == *k->word ) && !strcmp( k->word, buf ) )
- {
- yylval.type = k->type;
- yylval.keyword = k->word; /* used by symdump */
- break;
- }
-
- if ( yylval.type == ARG )
- yylval.string = object_new( buf );
- }
-
- if ( DEBUG_SCAN )
- printf( "scan %s\n", symdump( &yylval ) );
-
- return yylval.type;
-
-eof:
- /* We do not reset yylval.file & yylval.line here so unexpected EOF error
- * messages would include correct error location information.
- */
- yylval.type = EOF;
- return yylval.type;
-}
-
-
-static char * symdump( YYSTYPE * s )
-{
- static char buf[ BIGGEST_TOKEN + 20 ];
- switch ( s->type )
- {
- case EOF : sprintf( buf, "EOF" ); break;
- case 0 : sprintf( buf, "unknown symbol %s", object_str( s->string ) ); break;
- case ARG : sprintf( buf, "argument %s" , object_str( s->string ) ); break;
- case STRING: sprintf( buf, "string \"%s\"" , object_str( s->string ) ); break;
- default : sprintf( buf, "keyword %s" , s->keyword ); break;
- }
- return buf;
-}
-
-
-/*
- * Get information about the current file and line, for those epsilon
- * transitions that produce a parse.
- */
-
-void yyinput_last_read_token( OBJECT * * name, int * line )
-{
- /* TODO: Consider whether and when we might want to report where the last
- * read token ended, e.g. EOF errors inside string literals.
- */
- *name = yylval.file;
- *line = yylval.line;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/scan.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/scan.h b/ext/kenlm/jam-files/engine/scan.h
deleted file mode 100644
index 745477f..0000000
--- a/ext/kenlm/jam-files/engine/scan.h
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Copyright 1993, 1995 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/*
- * scan.h - the jam yacc scanner
- *
- * External functions:
- * yyerror( char *s ) - print a parsing error message.
- * yyfparse( char *s ) - scan include file s.
- * yylex() - parse the next token, returning its type.
- * yymode() - adjust lexicon of scanner.
- * yyparse() - declaration for yacc parser.
- * yyanyerrors() - indicate if any parsing errors occured.
- *
- * The yymode() function is for the parser to adjust the lexicon of the scanner.
- * Aside from normal keyword scanning, there is a mode to handle action strings
- * (look only for the closing }) and a mode to ignore most keywords when looking
- * for a punctuation keyword. This allows non-punctuation keywords to be used in
- * lists without quoting.
- */
-
-#include "lists.h"
-#include "object.h"
-#include "parse.h"
-
-
-/*
- * YYSTYPE - value of a lexical token
- */
-
-#define YYSTYPE YYSYMBOL
-
-typedef struct _YYSTYPE
-{
- int type;
- OBJECT * string;
- PARSE * parse;
- LIST * list;
- int number;
- OBJECT * file;
- int line;
- char const * keyword;
-} YYSTYPE;
-
-extern YYSTYPE yylval;
-
-void yymode( int n );
-void yyerror( char const * s );
-int yyanyerrors();
-void yyfparse( OBJECT * s );
-int yyline();
-int yylex();
-int yyparse();
-void yyinput_last_read_token( OBJECT * * name, int * line );
-
-#define SCAN_NORMAL 0 /* normal parsing */
-#define SCAN_STRING 1 /* look only for matching } */
-#define SCAN_PUNCT 2 /* only punctuation keywords */
[12/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/CMakeLists.txt b/ext/kenlm/lm/interpolate/CMakeLists.txt
deleted file mode 100644
index c146ab6..0000000
--- a/ext/kenlm/lm/interpolate/CMakeLists.txt
+++ /dev/null
@@ -1,54 +0,0 @@
-find_package(Eigen3 REQUIRED)
-include_directories(${EIGEN3_INCLUDE_DIR})
-
-set(KENLM_INTERPOLATE_SOURCE
- backoff_reunification.cc
- bounded_sequence_encoding.cc
- enumerate_global_vocab.cc
- merge_probabilities.cc
- merge_vocab.cc
- normalize.cc
- pipeline.cc
- split_worker.cc
- tune_derivatives.cc
- tune_instance.cc
- universal_vocab.cc)
-
-add_library(kenlm_interpolate OBJECT ${KENLM_INTERPOLATE_SOURCE})
-
-set(KENLM_INTERPOLATE_EXES
- interpolate
- perf_enum_gv
- streaming_example
- train_params
- tune)
-
-set(KENLM_INTERPOLATE_DEPENDS
- $<TARGET_OBJECTS:kenlm>
- $<TARGET_OBJECTS:kenlm_util>
- $<TARGET_OBJECTS:kenlm_common>
- $<TARGET_OBJECTS:kenlm_interpolate>)
-
-AddExes(EXES ${KENLM_INTERPOLATE_EXES}
- DEPENDS ${KENLM_INTERPOLATE_DEPENDS}
- LIBRARIES ${Boost_LIBRARIES} pthread)
-
-if(BUILD_TESTING)
- set(KENLM_INTERPOLATE_TESTS
- backoff_reunification_test
- bounded_sequence_encoding_test
- merge_vocab_test
- normalize_test
- tune_derivatives_test)
-
- AddTests(TESTS ${KENLM_INTERPOLATE_TESTS}
- DEPENDS ${KENLM_INTERPOLATE_DEPENDS}
- LIBRARIES ${Boost_LIBRARIES} pthread)
-
- # tune_instance_test needs an extra command line parameter
- KenLMAddTest(TEST tune_instance_test
- DEPENDS ${KENLM_INTERPOLATE_DEPENDS}
- LIBRARIES ${Boost_LIBRARIES} pthread
- TEST_ARGS
- ${CMAKE_CURRENT_SOURCE_DIR}/tune_instance_data/toy0.1)
-endif()
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/Jamfile
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/Jamfile b/ext/kenlm/lm/interpolate/Jamfile
deleted file mode 100644
index 411a346..0000000
--- a/ext/kenlm/lm/interpolate/Jamfile
+++ /dev/null
@@ -1,22 +0,0 @@
-fakelib interp : ../common//common [ glob *.cc : *_main.cc *_test.cc tune_*.cc ] : <cxxflags>-fopenmp ;
-
-import testing ;
-
-local with-eigen = [ option.get "with-eigen" ] ;
-if $(with-eigen) {
- fakelib tuning : tune_instance.cc tune_derivatives.cc interp ..//kenlm : <include>$(with-eigen) ;
- unit-test tune_derivatives_test : tune_derivatives_test.cc tuning /top//boost_unit_test_framework : <include>$(with-eigen) ;
-
- obj tune_instance_test.o : tune_instance_test.cc /top//boost_unit_test_framework : <include>$(with-eigen) ;
- run tune_instance_test.o tuning /top//boost_unit_test_framework : : tune_instance_data/toy0.1 ;
-
- exe tune : tune_main.cc tuning /top//boost_program_options : <include>$(with-eigen) ;
-}
-
-exe interpolate : interpolate_main.cc interp /top//boost_program_options ;
-exe streaming_example : ../builder//builder interp streaming_example_main.cc /top//boost_program_options ;
-
-unit-test normalize_test : interp normalize_test.cc /top//boost_unit_test_framework ;
-unit-test backoff_reunification_test : interp backoff_reunification_test.cc /top//boost_unit_test_framework ;
-unit-test bounded_sequence_encoding_test : interp bounded_sequence_encoding_test.cc /top//boost_unit_test_framework ;
-run merge_vocab_test.cc interp /top//boost_unit_test_framework : : merge_test/test1 merge_test/test2 merge_test/test3 merge_test/test_bad_order merge_test/test_no_unk ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/backoff_matrix.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/backoff_matrix.hh b/ext/kenlm/lm/interpolate/backoff_matrix.hh
deleted file mode 100644
index c7552df..0000000
--- a/ext/kenlm/lm/interpolate/backoff_matrix.hh
+++ /dev/null
@@ -1,29 +0,0 @@
-#ifndef LM_INTERPOLATE_BACKOFF_MATRIX_H
-#define LM_INTERPOLATE_BACKOFF_MATRIX_H
-
-#include <cstddef>
-#include <vector>
-
-namespace lm { namespace interpolate {
-
-class BackoffMatrix {
- public:
- BackoffMatrix(std::size_t num_models, std::size_t max_order)
- : max_order_(max_order), backing_(num_models * max_order) {}
-
- float &Backoff(std::size_t model, std::size_t order_minus_1) {
- return backing_[model * max_order_ + order_minus_1];
- }
-
- float Backoff(std::size_t model, std::size_t order_minus_1) const {
- return backing_[model * max_order_ + order_minus_1];
- }
-
- private:
- const std::size_t max_order_;
- std::vector<float> backing_;
-};
-
-}} // namespaces
-
-#endif // LM_INTERPOLATE_BACKOFF_MATRIX_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/backoff_reunification.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/backoff_reunification.cc b/ext/kenlm/lm/interpolate/backoff_reunification.cc
deleted file mode 100644
index edf6ddc..0000000
--- a/ext/kenlm/lm/interpolate/backoff_reunification.cc
+++ /dev/null
@@ -1,57 +0,0 @@
-#include "lm/interpolate/backoff_reunification.hh"
-#include "lm/common/model_buffer.hh"
-#include "lm/common/ngram_stream.hh"
-#include "lm/common/ngram.hh"
-#include "lm/common/compare.hh"
-
-#include <cassert>
-
-namespace lm {
-namespace interpolate {
-
-namespace {
-class MergeWorker {
-public:
- MergeWorker(std::size_t order, const util::stream::ChainPosition &prob_pos,
- const util::stream::ChainPosition &boff_pos)
- : order_(order), prob_pos_(prob_pos), boff_pos_(boff_pos) {
- // nothing
- }
-
- void Run(const util::stream::ChainPosition &position) {
- lm::NGramStream<ProbBackoff> stream(position);
-
- lm::NGramStream<float> prob_input(prob_pos_);
- util::stream::Stream boff_input(boff_pos_);
- for (; prob_input && boff_input; ++prob_input, ++boff_input, ++stream) {
- std::copy(prob_input->begin(), prob_input->end(), stream->begin());
- stream->Value().prob = prob_input->Value();
- stream->Value().backoff = *reinterpret_cast<float *>(boff_input.Get());
- }
- UTIL_THROW_IF2(prob_input || boff_input,
- "Streams were not the same size during merging");
- stream.Poison();
- }
-
-private:
- std::size_t order_;
- util::stream::ChainPosition prob_pos_;
- util::stream::ChainPosition boff_pos_;
-};
-}
-
-// Since we are *adding* something to the output chain here, we pass in the
-// chain itself so that we can safely add a new step to the chain without
-// creating a deadlock situation (since creating a new ChainPosition will
-// make a new input/output pair---we want that position to be created
-// *here*, not before).
-void ReunifyBackoff(util::stream::ChainPositions &prob_pos,
- util::stream::ChainPositions &boff_pos,
- util::stream::Chains &output_chains) {
- assert(prob_pos.size() == boff_pos.size());
-
- for (size_t i = 0; i < prob_pos.size(); ++i)
- output_chains[i] >> MergeWorker(i + 1, prob_pos[i], boff_pos[i]);
-}
-}
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/backoff_reunification.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/backoff_reunification.hh b/ext/kenlm/lm/interpolate/backoff_reunification.hh
deleted file mode 100644
index 327db65..0000000
--- a/ext/kenlm/lm/interpolate/backoff_reunification.hh
+++ /dev/null
@@ -1,27 +0,0 @@
-#ifndef KENLM_INTERPOLATE_BACKOFF_REUNIFICATION_
-#define KENLM_INTERPOLATE_BACKOFF_REUNIFICATION_
-
-#include "util/stream/stream.hh"
-#include "util/stream/multi_stream.hh"
-
-namespace lm {
-namespace interpolate {
-
-/**
- * The third pass for the offline log-linear interpolation algorithm. This
- * reads **suffix-ordered** probability values (ngram-id, float) and
- * **suffix-ordered** backoff values (float) and writes the merged contents
- * to the output.
- *
- * @param prob_pos The chain position for each order from which to read
- * the probability values
- * @param boff_pos The chain position for each order from which to read
- * the backoff values
- * @param output_chains The output chains for each order
- */
-void ReunifyBackoff(util::stream::ChainPositions &prob_pos,
- util::stream::ChainPositions &boff_pos,
- util::stream::Chains &output_chains);
-}
-}
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/backoff_reunification_test.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/backoff_reunification_test.cc b/ext/kenlm/lm/interpolate/backoff_reunification_test.cc
deleted file mode 100644
index d8faf94..0000000
--- a/ext/kenlm/lm/interpolate/backoff_reunification_test.cc
+++ /dev/null
@@ -1,159 +0,0 @@
-#include "lm/interpolate/backoff_reunification.hh"
-#include "lm/common/ngram_stream.hh"
-
-#define BOOST_TEST_MODULE InterpolateBackoffReunificationTest
-#include <boost/test/unit_test.hpp>
-
-namespace lm {
-namespace interpolate {
-
-namespace {
-
-// none of this input actually makes sense, all we care about is making
-// sure the merging works
-template <uint8_t N>
-struct Gram {
- WordIndex ids[N];
- float prob;
- float boff;
-};
-
-template <uint8_t N>
-struct Grams {
- const static Gram<N> grams[];
-};
-
-template <>
-const Gram<1> Grams<1>::grams[]
- = {{{0}, 0.1f, 0.1f}, {{1}, 0.4f, 0.2f}, {{2}, 0.5f, 0.1f}};
-
-template <>
-const Gram<2> Grams<2>::grams[] = {{{0, 0}, 0.05f, 0.05f},
- {{1, 0}, 0.05f, 0.02f},
- {{1, 1}, 0.2f, 0.04f},
- {{2, 2}, 0.2f, 0.01f}};
-
-template <>
-const Gram<3> Grams<3>::grams[] = {{{0, 0, 0}, 0.001f, 0.005f},
- {{1, 0, 0}, 0.001f, 0.002f},
- {{2, 0, 0}, 0.001f, 0.003f},
- {{0, 1, 0}, 0.1f, 0.008f},
- {{1, 1, 0}, 0.1f, 0.09f},
- {{1, 1, 1}, 0.2f, 0.08f}};
-
-template <uint8_t N>
-class WriteInput {
-public:
- void Run(const util::stream::ChainPosition &position) {
- lm::NGramStream<float> output(position);
-
- for (std::size_t i = 0; i < sizeof(Grams<N>::grams) / sizeof(Gram<N>);
- ++i, ++output) {
- std::copy(Grams<N>::grams[i].ids, Grams<N>::grams[i].ids + N,
- output->begin());
- output->Value() = Grams<N>::grams[i].prob;
- }
- output.Poison();
- }
-};
-
-template <uint8_t N>
-class WriteBackoffs {
-public:
- void Run(const util::stream::ChainPosition &position) {
- util::stream::Stream output(position);
-
- for (std::size_t i = 0; i < sizeof(Grams<N>::grams) / sizeof(Gram<N>);
- ++i, ++output) {
- *reinterpret_cast<float *>(output.Get()) = Grams<N>::grams[i].boff;
- }
- output.Poison();
- }
-};
-
-template <uint8_t N>
-class CheckOutput {
-public:
- void Run(const util::stream::ChainPosition &position) {
- lm::NGramStream<ProbBackoff> stream(position);
-
- std::size_t i = 0;
- for (; stream; ++stream, ++i) {
- std::stringstream ss;
- for (WordIndex *idx = stream->begin(); idx != stream->end(); ++idx)
- ss << "(" << *idx << ")";
-
- UTIL_THROW_IF2(
- !std::equal(stream->begin(), stream->end(), Grams<N>::grams[i].ids),
- "Mismatched id in CheckOutput<" << (int)N << ">: " << ss.str());
-
- UTIL_THROW_IF2(stream->Value().prob != Grams<N>::grams[i].prob,
- "Mismatched probability in CheckOutput<"
- << (int)N << ">, got " << stream->Value().prob
- << ", expected " << Grams<N>::grams[i].prob);
-
- UTIL_THROW_IF2(stream->Value().backoff != Grams<N>::grams[i].boff,
- "Mismatched backoff in CheckOutput<"
- << (int)N << ">, got " << stream->Value().backoff
- << ", expected " << Grams<N>::grams[i].boff);
- }
- UTIL_THROW_IF2(i != sizeof(Grams<N>::grams) / sizeof(Gram<N>),
- "Did not get correct number of "
- << (int)N << "-grams: expected "
- << sizeof(Grams<N>::grams) / sizeof(Gram<N>)
- << ", got " << i);
- }
-};
-}
-
-BOOST_AUTO_TEST_CASE(BackoffReunificationTest) {
- util::stream::ChainConfig config;
- config.total_memory = 100;
- config.block_count = 1;
-
- util::stream::Chains prob_chains(3);
- config.entry_size = NGram<float>::TotalSize(1);
- prob_chains.push_back(config);
- prob_chains.back() >> WriteInput<1>();
-
- config.entry_size = NGram<float>::TotalSize(2);
- prob_chains.push_back(config);
- prob_chains.back() >> WriteInput<2>();
-
- config.entry_size = NGram<float>::TotalSize(3);
- prob_chains.push_back(config);
- prob_chains.back() >> WriteInput<3>();
-
- util::stream::Chains boff_chains(3);
- config.entry_size = sizeof(float);
- boff_chains.push_back(config);
- boff_chains.back() >> WriteBackoffs<1>();
-
- boff_chains.push_back(config);
- boff_chains.back() >> WriteBackoffs<2>();
-
- boff_chains.push_back(config);
- boff_chains.back() >> WriteBackoffs<3>();
-
- util::stream::ChainPositions prob_pos(prob_chains);
- util::stream::ChainPositions boff_pos(boff_chains);
-
- util::stream::Chains output_chains(3);
- for (std::size_t i = 0; i < 3; ++i) {
- config.entry_size = NGram<ProbBackoff>::TotalSize(i + 1);
- output_chains.push_back(config);
- }
-
- ReunifyBackoff(prob_pos, boff_pos, output_chains);
-
- output_chains[0] >> CheckOutput<1>();
- output_chains[1] >> CheckOutput<2>();
- output_chains[2] >> CheckOutput<3>();
-
- prob_chains >> util::stream::kRecycle;
- boff_chains >> util::stream::kRecycle;
-
- output_chains.Wait();
-}
-}
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/bounded_sequence_encoding.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/bounded_sequence_encoding.cc b/ext/kenlm/lm/interpolate/bounded_sequence_encoding.cc
deleted file mode 100644
index aca8ed7..0000000
--- a/ext/kenlm/lm/interpolate/bounded_sequence_encoding.cc
+++ /dev/null
@@ -1,36 +0,0 @@
-#include "lm/interpolate/bounded_sequence_encoding.hh"
-
-#include <algorithm>
-
-namespace lm { namespace interpolate {
-
-BoundedSequenceEncoding::BoundedSequenceEncoding(const unsigned char *bound_begin, const unsigned char *bound_end)
- : entries_(bound_end - bound_begin) {
- std::size_t full = 0;
- Entry entry;
- entry.shift = 0;
- for (const unsigned char *i = bound_begin; i != bound_end; ++i) {
- uint8_t length;
- if (*i <= 1) {
- length = 0;
- } else {
- length = sizeof(unsigned int) * 8 - __builtin_clz((unsigned int)*i);
- }
- entry.mask = (1ULL << length) - 1ULL;
- if (entry.shift + length > 64) {
- entry.shift = 0;
- entry.next = true;
- ++full;
- } else {
- entry.next = false;
- }
- entries_.push_back(entry);
- entry.shift += length;
- }
- byte_length_ = full * sizeof(uint64_t) + (entry.shift + 7) / 8;
- first_copy_ = std::min<std::size_t>(byte_length_, sizeof(uint64_t));
- // Size of last uint64_t. Zero if empty, otherwise [1,8] depending on mod.
- overhang_ = byte_length_ == 0 ? 0 : ((byte_length_ - 1) % 8 + 1);
-}
-
-}} // namespaces
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/bounded_sequence_encoding.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/bounded_sequence_encoding.hh b/ext/kenlm/lm/interpolate/bounded_sequence_encoding.hh
deleted file mode 100644
index 84dd63a..0000000
--- a/ext/kenlm/lm/interpolate/bounded_sequence_encoding.hh
+++ /dev/null
@@ -1,76 +0,0 @@
-#ifndef LM_INTERPOLATE_BOUNDED_SEQUENCE_ENCODING_H
-#define LM_INTERPOLATE_BOUNDED_SEQUENCE_ENCODING_H
-
-/* Encodes fixed-length sequences of integers with known bounds on each entry.
- * This is used to encode how far each model has backed off.
- * TODO: make this class efficient. Bit-level packing or multiply by bound and
- * add.
- */
-
-#include "util/exception.hh"
-#include "util/fixed_array.hh"
-
-#if BYTE_ORDER != LITTLE_ENDIAN
-#warning The interpolation code assumes little endian for now.
-#endif
-
-#include <algorithm>
-#include <cstring>
-
-namespace lm {
-namespace interpolate {
-
-class BoundedSequenceEncoding {
- public:
- // Encode [0, bound_begin[0]) x [0, bound_begin[1]) x [0, bound_begin[2]) x ... x [0, *(bound_end - 1)) for entries in the sequence
- BoundedSequenceEncoding(const unsigned char *bound_begin, const unsigned char *bound_end);
-
- std::size_t Entries() const { return entries_.size(); }
-
- std::size_t EncodedLength() const { return byte_length_; }
-
- void Encode(const unsigned char *from, void *to_void) const {
- uint8_t *to = static_cast<uint8_t*>(to_void);
- uint64_t cur = 0;
- for (const Entry *i = entries_.begin(); i != entries_.end(); ++i, ++from) {
- if (UTIL_UNLIKELY(i->next)) {
- std::memcpy(to, &cur, sizeof(uint64_t));
- to += sizeof(uint64_t);
- cur = 0;
- }
- cur |= static_cast<uint64_t>(*from) << i->shift;
- }
- memcpy(to, &cur, overhang_);
- }
-
- void Decode(const void *from_void, unsigned char *to) const {
- const uint8_t *from = static_cast<const uint8_t*>(from_void);
- uint64_t cur = 0;
- memcpy(&cur, from, first_copy_);
- for (const Entry *i = entries_.begin(); i != entries_.end(); ++i, ++to) {
- if (UTIL_UNLIKELY(i->next)) {
- from += sizeof(uint64_t);
- cur = 0;
- std::memcpy(&cur, from,
- std::min<std::size_t>(sizeof(uint64_t), static_cast<const uint8_t*>(from_void) + byte_length_ - from));
- }
- *to = (cur >> i->shift) & i->mask;
- }
- }
-
- private:
- struct Entry {
- bool next;
- uint8_t shift;
- uint64_t mask;
- };
- util::FixedArray<Entry> entries_;
- std::size_t byte_length_;
- std::size_t first_copy_;
- std::size_t overhang_;
-};
-
-
-}} // namespaces
-
-#endif // LM_INTERPOLATE_BOUNDED_SEQUENCE_ENCODING_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/bounded_sequence_encoding_test.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/bounded_sequence_encoding_test.cc b/ext/kenlm/lm/interpolate/bounded_sequence_encoding_test.cc
deleted file mode 100644
index 2c4bbd9..0000000
--- a/ext/kenlm/lm/interpolate/bounded_sequence_encoding_test.cc
+++ /dev/null
@@ -1,86 +0,0 @@
-#include "lm/interpolate/bounded_sequence_encoding.hh"
-
-#include "util/scoped.hh"
-
-#define BOOST_TEST_MODULE BoundedSequenceEncodingTest
-#include <boost/test/unit_test.hpp>
-
-namespace lm {
-namespace interpolate {
-namespace {
-
-void ExhaustiveTest(unsigned char *bound_begin, unsigned char *bound_end) {
- BoundedSequenceEncoding enc(bound_begin, bound_end);
- util::scoped_malloc backing(util::MallocOrThrow(enc.EncodedLength()));
- std::vector<unsigned char> values(bound_end - bound_begin),
- out(bound_end - bound_begin);
- while (true) {
- enc.Encode(&values[0], backing.get());
- enc.Decode(backing.get(), &out[0]);
- for (std::size_t i = 0; i != values.size(); ++i) {
- BOOST_CHECK_EQUAL(values[i], out[i]);
- }
- for (std::size_t i = 0;; ++i) {
- if (i == values.size()) return;
- ++values[i];
- if (values[i] < bound_begin[i]) break;
- values[i] = 0;
- }
- }
-}
-
-void CheckEncodeDecode(unsigned char *bounds, unsigned char *input,
- unsigned char *output, std::size_t len) {
- BoundedSequenceEncoding encoder(bounds, bounds + len);
- util::scoped_malloc backing(util::MallocOrThrow(encoder.EncodedLength()));
-
- encoder.Encode(input, backing.get());
- encoder.Decode(backing.get(), output);
-
- for (std::size_t i = 0; i < len; ++i) {
- BOOST_CHECK_EQUAL(input[i], output[i]);
- }
-}
-
-BOOST_AUTO_TEST_CASE(Exhaustive) {
- unsigned char bounds[] = {5, 2, 3, 9, 7, 20, 8};
- ExhaustiveTest(bounds, bounds + sizeof(bounds) / sizeof(unsigned char));
-}
-
-BOOST_AUTO_TEST_CASE(LessThan64) {
- unsigned char bounds[] = {255, 255, 255, 255, 255, 255, 255, 3};
- unsigned char input[] = {172, 183, 254, 187, 96, 87, 65, 2};
- unsigned char output[] = {0, 0, 0, 0, 0, 0, 0, 0};
-
- std::size_t len = sizeof(bounds) / sizeof(unsigned char);
- assert(sizeof(input) / sizeof(unsigned char) == len);
- assert(sizeof(output) / sizeof(unsigned char) == len);
-
- CheckEncodeDecode(bounds, input, output, len);
-}
-
-BOOST_AUTO_TEST_CASE(Exactly64) {
- unsigned char bounds[] = {255, 255, 255, 255, 255, 255, 255, 255};
- unsigned char input[] = {172, 183, 254, 187, 96, 87, 65, 16};
- unsigned char output[] = {0, 0, 0, 0, 0, 0, 0, 0};
-
- std::size_t len = sizeof(bounds) / sizeof(unsigned char);
- assert(sizeof(input) / sizeof(unsigned char) == len);
- assert(sizeof(output) / sizeof(unsigned char) == len);
-
- CheckEncodeDecode(bounds, input, output, len);
-}
-
-BOOST_AUTO_TEST_CASE(MoreThan64) {
- unsigned char bounds[] = {255, 255, 255, 255, 255, 255, 255, 255, 255};
- unsigned char input[] = {172, 183, 254, 187, 96, 87, 65, 16, 137};
- unsigned char output[] = {0, 0, 0, 0, 0, 0, 0, 0, 0};
-
- std::size_t len = sizeof(bounds) / sizeof(unsigned char);
- assert(sizeof(input) / sizeof(unsigned char) == len);
- assert(sizeof(output) / sizeof(unsigned char) == len);
-
- CheckEncodeDecode(bounds, input, output, len);
-}
-
-}}} // namespaces
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/enumerate_global_vocab.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/enumerate_global_vocab.cc b/ext/kenlm/lm/interpolate/enumerate_global_vocab.cc
deleted file mode 100644
index 26f63c8..0000000
--- a/ext/kenlm/lm/interpolate/enumerate_global_vocab.cc
+++ /dev/null
@@ -1,48 +0,0 @@
-
-#include "lm/interpolate/enumerate_global_vocab.hh"
-
-#include <iostream>
-#include <map>
-
-namespace lm {
-//constructor
-
- EnumerateGlobalVocab::EnumerateGlobalVocab(std::map<std::string, int*> * vm, int nm) {
-
- vmap = vm;
- num_models = nm;
- cur_model = 0; //blah
- cnt = 0;
- std::cerr << "Vocab Builder with models: " << nm << std::endl;
- }
-
- void EnumerateGlobalVocab::Add(WordIndex index, const StringPiece &str) {
-
- std::string st = str.as_string();
-
- //check for existence of key
- std::map<std::string, int*>::iterator itr = vmap->find(st);
-
- //put stuff
- if(itr != vmap->end()) {
- std::cerr << "Vocab exist: " << str << " M: " << cur_model << " I:" << index << std::endl;
- itr->second[cur_model] = index;
- }
- //new key
- else {
-
- //create model index map for this vocab word
- //init to 0, 0 is UNK
- int * indices = new int[num_models];
- memset(indices, 0, (sizeof(int)*num_models)); //this still legit?
-
- indices[cur_model] = index;
- (*vmap)[st] = indices;
- std::cerr << cnt << ":Vocab add: " << str << " M: " << cur_model << " I:" << index << std::endl;
- cnt++;
- }
-
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/enumerate_global_vocab.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/enumerate_global_vocab.hh b/ext/kenlm/lm/interpolate/enumerate_global_vocab.hh
deleted file mode 100644
index c37d649..0000000
--- a/ext/kenlm/lm/interpolate/enumerate_global_vocab.hh
+++ /dev/null
@@ -1,38 +0,0 @@
-#ifndef LM_ENUMERATE_GLOBAL_VOCAB_H
-#define LM_ENUMERATE_GLOBAL_VOCAB_H
-
-#include "lm/enumerate_vocab.hh"
-#include <map>
-
-/* Use this to create a global vocab across models for use when
- * calculating lambdas for interpolation. Or other stuff.
- */
-namespace lm {
-
- class EnumerateGlobalVocab : EnumerateVocab {
-
- public:
-
- //yes, ugly...
- std::map<std::string, int*> * vmap;
- int num_models;
- int cur_model;
- int cnt; //stupid
-
- ~EnumerateGlobalVocab() {}
-
- void Add(WordIndex index, const StringPiece & str);
-
- EnumerateGlobalVocab(std::map<std::string, int*> *, int);
-
- void SetCurModel(int i) { cur_model = i; }
-
- protected:
- EnumerateGlobalVocab() {}
-
- };
-
-} //namespace lm
-
-#endif // LM_ENUMERATE_GLOBAL_VOCAB_H
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/interpolate_info.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/interpolate_info.hh b/ext/kenlm/lm/interpolate/interpolate_info.hh
deleted file mode 100644
index ebecd92..0000000
--- a/ext/kenlm/lm/interpolate/interpolate_info.hh
+++ /dev/null
@@ -1,35 +0,0 @@
-#ifndef KENLM_INTERPOLATE_INTERPOLATE_INFO_H
-#define KENLM_INTERPOLATE_INTERPOLATE_INFO_H
-
-#include <cstddef>
-#include <vector>
-#include <stdint.h>
-
-namespace lm {
-namespace interpolate {
-
-/**
- * Stores relevant info for interpolating several language models, for use
- * during the three-pass offline log-linear interpolation algorithm.
- */
-struct InterpolateInfo {
- /**
- * @return the number of models being interpolated
- */
- std::size_t Models() const {
- return orders.size();
- }
-
- /**
- * The lambda (interpolation weight) for each model.
- */
- std::vector<float> lambdas;
-
- /**
- * The maximum ngram order for each model.
- */
- std::vector<uint8_t> orders;
-};
-}
-}
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/interpolate_main.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/interpolate_main.cc b/ext/kenlm/lm/interpolate/interpolate_main.cc
deleted file mode 100644
index a99b62d..0000000
--- a/ext/kenlm/lm/interpolate/interpolate_main.cc
+++ /dev/null
@@ -1,37 +0,0 @@
-#include "lm/common/model_buffer.hh"
-#include "lm/common/size_option.hh"
-#include "lm/interpolate/pipeline.hh"
-#include "util/fixed_array.hh"
-#include "util/usage.hh"
-
-#include <boost/program_options.hpp>
-
-#include <iostream>
-#include <vector>
-
-int main(int argc, char *argv[]) {
- lm::interpolate::Config config;
- std::vector<std::string> input_models;
- namespace po = boost::program_options;
- po::options_description options("Log-linear interpolation options");
- options.add_options()
- ("lambda,w", po::value<std::vector<float> >(&config.lambdas)->multitoken()->required(), "Interpolation weights")
- ("model,m", po::value<std::vector<std::string> >(&input_models)->multitoken()->required(), "Models to interpolate")
- ("temp_prefix,T", po::value<std::string>(&config.sort.temp_prefix)->default_value("/tmp/lm"), "Temporary file prefix")
- ("memory,S", lm::SizeOption(config.sort.total_memory, util::GuessPhysicalMemory() ? "50%" : "1G"), "Sorting memory")
- ("sort_block", lm::SizeOption(config.sort.buffer_size, "64M"), "Block size");
- po::variables_map vm;
- po::store(po::parse_command_line(argc, argv, options), vm);
- po::notify(vm);
-
- if (config.lambdas.size() != input_models.size()) {
- std::cerr << "Number of models " << input_models.size() << " should match the number of weights" << config.lambdas.size() << "." << std::endl;
- return 1;
- }
-
- util::FixedArray<lm::ModelBuffer> models(input_models.size());
- for (std::size_t i = 0; i < input_models.size(); ++i) {
- models.push_back(input_models[i]);
- }
- lm::interpolate::Pipeline(models, config, 1);
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/merge_probabilities.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/merge_probabilities.cc b/ext/kenlm/lm/interpolate/merge_probabilities.cc
deleted file mode 100644
index b6c949f..0000000
--- a/ext/kenlm/lm/interpolate/merge_probabilities.cc
+++ /dev/null
@@ -1,285 +0,0 @@
-#include "lm/interpolate/merge_probabilities.hh"
-#include "lm/common/ngram_stream.hh"
-#include "lm/interpolate/bounded_sequence_encoding.hh"
-#include "lm/interpolate/interpolate_info.hh"
-
-#include <algorithm>
-#include <limits>
-#include <numeric>
-
-namespace lm {
-namespace interpolate {
-
-/**
- * Helper to generate the BoundedSequenceEncoding used for writing the
- * from values.
- */
-BoundedSequenceEncoding MakeEncoder(const InterpolateInfo &info, uint8_t order) {
- util::FixedArray<uint8_t> max_orders(info.orders.size());
- for (std::size_t i = 0; i < info.orders.size(); ++i) {
- max_orders.push_back(std::min(order, info.orders[i]));
- }
- return BoundedSequenceEncoding(max_orders.begin(), max_orders.end());
-}
-
-namespace {
-
-/**
- * A simple wrapper class that holds information needed to read and write
- * the ngrams of a particular order. This class has the memory needed to
- * buffer the data needed for the recursive process of computing the
- * probabilities and "from" values for each component model.
- *
- * "From" values indicate, for each model, what order (as an index, so -1)
- * was backed off to in order to arrive at a probability. For example, if a
- * 5-gram model (order index 4) backed off twice, we would write a 2.
- */
-class NGramHandler {
-public:
- NGramHandler(uint8_t order, const InterpolateInfo &ifo,
- util::FixedArray<util::stream::ChainPositions> &models_by_order)
- : info(ifo),
- encoder(MakeEncoder(info, order)),
- out_record(order, encoder.EncodedLength()) {
- std::size_t count_has_order = 0;
- for (std::size_t i = 0; i < models_by_order.size(); ++i) {
- count_has_order += (models_by_order[i].size() >= order);
- }
- inputs_.Init(count_has_order);
- for (std::size_t i = 0; i < models_by_order.size(); ++i) {
- if (models_by_order[i].size() < order)
- continue;
- inputs_.push_back(models_by_order[i][order - 1]);
- if (inputs_.back()) {
- active_.resize(active_.size() + 1);
- active_.back().model = i;
- active_.back().stream = &inputs_.back();
- }
- }
-
- // have to init outside since NGramStreams doesn't forward to
- // GenericStreams ctor given a ChainPositions
-
- probs.Init(info.Models());
- from.Init(info.Models());
- for (std::size_t i = 0; i < info.Models(); ++i) {
- probs.push_back(0.0);
- from.push_back(0);
- }
- }
-
- struct StreamIndex {
- NGramStream<ProbBackoff> *stream;
- NGramStream<ProbBackoff> &Stream() { return *stream; }
- std::size_t model;
- };
-
- std::size_t ActiveSize() const {
- return active_.size();
- }
-
- /**
- * @return the input stream for a particular model that corresponds to
- * this ngram order
- */
- StreamIndex &operator[](std::size_t idx) {
- return active_[idx];
- }
-
- void erase(std::size_t idx) {
- active_.erase(active_.begin() + idx);
- }
-
- const InterpolateInfo &info;
- BoundedSequenceEncoding encoder;
- PartialProbGamma out_record;
- util::FixedArray<float> probs;
- util::FixedArray<uint8_t> from;
-
-private:
- std::vector<StreamIndex> active_;
- NGramStreams<ProbBackoff> inputs_;
-};
-
-/**
- * A collection of NGramHandlers.
- */
-class NGramHandlers : public util::FixedArray<NGramHandler> {
-public:
- explicit NGramHandlers(std::size_t num)
- : util::FixedArray<NGramHandler>(num) {
- }
-
- void push_back(
- std::size_t order, const InterpolateInfo &info,
- util::FixedArray<util::stream::ChainPositions> &models_by_order) {
- new (end()) NGramHandler(order, info, models_by_order);
- Constructed();
- }
-};
-
-/**
- * The recursive helper function that computes probability and "from"
- * values for all ngrams matching a particular suffix.
- *
- * The current order can be computed as the suffix length + 1. Note that
- * the suffix could be empty (suffix_begin == suffix_end == NULL), in which
- * case we are handling unigrams with the UNK token as the fallback
- * probability.
- *
- * @param handlers The full collection of handlers
- * @param suffix_begin A start iterator for the suffix
- * @param suffix_end An end iterator for the suffix
- * @param fallback_probs The probabilities of this ngram if we need to
- * back off (that is, the probability of the suffix)
- * @param fallback_from The order that the corresponding fallback
- * probability in the fallback_probs is from
- * @param combined_fallback interpolated fallback_probs
- * @param outputs The output streams, one for each order
- */
-void HandleSuffix(NGramHandlers &handlers, WordIndex *suffix_begin,
- WordIndex *suffix_end,
- const util::FixedArray<float> &fallback_probs,
- const util::FixedArray<uint8_t> &fallback_from,
- float combined_fallback,
- util::stream::Streams &outputs) {
- uint8_t order = std::distance(suffix_begin, suffix_end) + 1;
- if (order > outputs.size()) return;
-
- util::stream::Stream &output = outputs[order - 1];
- NGramHandler &handler = handlers[order - 1];
-
- while (true) {
- // find the next smallest ngram which matches our suffix
- // TODO: priority queue driven.
- WordIndex *minimum = NULL;
- for (std::size_t i = 0; i < handler.ActiveSize(); ++i) {
- if (!std::equal(suffix_begin, suffix_end, handler[i].Stream()->begin() + 1))
- continue;
-
- // if we either haven't set a minimum yet or this one is smaller than
- // the minimum we found before, replace it
- WordIndex *last = handler[i].Stream()->begin();
- if (!minimum || *last < *minimum) { minimum = handler[i].Stream()->begin(); }
- }
-
- // no more ngrams of this order match our suffix, so we're done
- if (!minimum) return;
-
- handler.out_record.ReBase(output.Get());
- std::copy(minimum, minimum + order, handler.out_record.begin());
-
- // Default case is having backed off.
- std::copy(fallback_probs.begin(), fallback_probs.end(), handler.probs.begin());
- std::copy(fallback_from.begin(), fallback_from.end(), handler.from.begin());
-
- for (std::size_t i = 0; i < handler.ActiveSize();) {
- if (std::equal(handler.out_record.begin(), handler.out_record.end(),
- handler[i].Stream()->begin())) {
- handler.probs[handler[i].model] = handler.info.lambdas[handler[i].model] * handler[i].Stream()->Value().prob;
- handler.from[handler[i].model] = order - 1;
- if (++handler[i].Stream()) {
- ++i;
- } else {
- handler.erase(i);
- }
- } else {
- ++i;
- }
- }
- handler.out_record.Prob() = std::accumulate(handler.probs.begin(), handler.probs.end(), 0.0);
- handler.out_record.LowerProb() = combined_fallback;
- handler.encoder.Encode(handler.from.begin(),
- handler.out_record.FromBegin());
-
- // we've handled this particular ngram, so now recurse to the higher
- // order using the current ngram as the suffix
- HandleSuffix(handlers, handler.out_record.begin(), handler.out_record.end(),
- handler.probs, handler.from, handler.out_record.Prob(), outputs);
- // consume the output
- ++output;
- }
-}
-
-/**
- * Kicks off the recursion for computing the probabilities and "from"
- * values for each ngram order. We begin by handling the UNK token that
- * should be at the front of each of the unigram input streams. This is
- * then output to the stream and it is used as the fallback for handling
- * our unigram case, the unigram used as the fallback for the bigram case,
- * etc.
- */
-void HandleNGrams(NGramHandlers &handlers, util::stream::Streams &outputs) {
- PartialProbGamma unk_record(1, 0);
- // First: populate the unk probabilities by reading the first unigram
- // from each stream
- util::FixedArray<float> unk_probs(handlers[0].info.Models());
-
- // start by populating the ngram id from the first stream
- lm::NGram<ProbBackoff> ngram = *handlers[0][0].Stream();
- unk_record.ReBase(outputs[0].Get());
- std::copy(ngram.begin(), ngram.end(), unk_record.begin());
- unk_record.Prob() = 0;
-
- // then populate the probabilities into unk_probs while "multiply" the
- // model probabilities together into the unk record
- //
- // note that from doesn't need to be set for unigrams
- assert(handlers[0].ActiveSize() == handlers[0].info.Models());
- for (std::size_t i = 0; i < handlers[0].info.Models();) {
- ngram = *handlers[0][i].Stream();
- unk_probs.push_back(handlers[0].info.lambdas[i] * ngram.Value().prob);
- unk_record.Prob() += unk_probs[i];
- assert(*ngram.begin() == kUNK);
- if (++handlers[0][i].Stream()) {
- ++i;
- } else {
- handlers[0].erase(i);
- }
- }
- float unk_combined = unk_record.Prob();
- unk_record.LowerProb() = unk_combined;
- // flush the unk output record
- ++outputs[0];
-
- // Then, begin outputting everything in lexicographic order: first we'll
- // get the unigram then the first bigram with that context, then the
- // first trigram with that bigram context, etc., until we exhaust all of
- // the ngrams, then all of the (n-1)grams, etc.
- //
- // This function is the "root" of this recursive process.
- util::FixedArray<uint8_t> unk_from(handlers[0].info.Models());
- for (std::size_t i = 0; i < handlers[0].info.Models(); ++i) {
- unk_from.push_back(0);
- }
-
- // the two nulls are to encode that our "fallback" word is the "0-gram"
- // case, e.g. we "backed off" to UNK
- // TODO: stop generating vocab ids and LowerProb for unigrams.
- HandleSuffix(handlers, NULL, NULL, unk_probs, unk_from, unk_combined, outputs);
-
- // Verify we reached the end. And poison!
- for (std::size_t i = 0; i < handlers.size(); ++i) {
- UTIL_THROW_IF2(handlers[i].ActiveSize(),
- "MergeProbabilities did not exhaust all ngram streams");
- outputs[i].Poison();
- }
-}
-}
-
-void MergeProbabilities(
- const InterpolateInfo &info,
- util::FixedArray<util::stream::ChainPositions> &models_by_order,
- util::stream::Chains &output_chains) {
- NGramHandlers handlers(output_chains.size());
- for (std::size_t i = 0; i < output_chains.size(); ++i) {
- handlers.push_back(i + 1, info, models_by_order);
- }
-
- util::stream::ChainPositions output_pos(output_chains);
- util::stream::Streams outputs(output_pos);
-
- HandleNGrams(handlers, outputs);
-}
-}
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/merge_probabilities.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/merge_probabilities.hh b/ext/kenlm/lm/interpolate/merge_probabilities.hh
deleted file mode 100644
index 59b8046..0000000
--- a/ext/kenlm/lm/interpolate/merge_probabilities.hh
+++ /dev/null
@@ -1,89 +0,0 @@
-#ifndef LM_INTERPOLATE_MERGE_PROBABILITIES_H
-#define LM_INTERPOLATE_MERGE_PROBABILITIES_H
-
-#include "lm/common/ngram.hh"
-#include "lm/interpolate/bounded_sequence_encoding.hh"
-#include "util/fixed_array.hh"
-#include "util/stream/multi_stream.hh"
-
-#include <stdint.h>
-
-namespace lm {
-namespace interpolate {
-
-struct InterpolateInfo;
-
-/**
- * Make the encoding of backoff values for a given order. This stores values
- * in [PartialProbGamma::FromBegin(), PartialProbGamma::FromEnd())
- */
-BoundedSequenceEncoding MakeEncoder(const InterpolateInfo &info, uint8_t order);
-
-/**
- * The first pass for the offline log-linear interpolation algorithm. This
- * reads K **suffix-ordered** streams for each model, for each order, of
- * ngram records (ngram-id, prob, backoff). It further assumes that the
- * ngram-ids have been unified over all of the stream inputs.
- *
- * Its output is records of (ngram-id, prob-prod, backoff-level,
- * backoff-level, ...) where the backoff-levels (of which there are K) are
- * the context length (0 for unigrams) that the corresponding model had to
- * back off to in order to obtain a probability for that ngram-id. Each of
- * these streams is terminated with a record whose ngram-id is all
- * maximum-integers for simplicity in implementation here.
- *
- * @param models An array of length N (max_i N_i) containing at
- * the ChainPositions for the streams for order (i + 1).
- * @param output_chains The output chains for each order (of length K)
- */
-void MergeProbabilities(
- const InterpolateInfo &info,
- util::FixedArray<util::stream::ChainPositions> &models_by_order,
- util::stream::Chains &output_chains);
-
-/**
- * This class represents the output payload for this pass, which consists
- * of an ngram-id, a probability, and then a vector of orders from which
- * each of the component models backed off to for this ngram, encoded
- * using the BoundedSequenceEncoding class.
- */
-class PartialProbGamma : public lm::NGramHeader {
-public:
- PartialProbGamma(std::size_t order, std::size_t backoff_bytes)
- : lm::NGramHeader(NULL, order), backoff_bytes_(backoff_bytes) {
- // nothing
- }
-
- std::size_t TotalSize() const {
- return sizeof(WordIndex) * Order() + sizeof(After) + backoff_bytes_;
- }
-
- // TODO: cache bounded sequence encoding in the pipeline?
- static std::size_t TotalSize(const InterpolateInfo &info, uint8_t order) {
- return sizeof(WordIndex) * order + sizeof(After) + MakeEncoder(info, order).EncodedLength();
- }
-
- float &Prob() { return Pay().prob; }
- float Prob() const { return Pay().prob; }
-
- float &LowerProb() { return Pay().lower_prob; }
- float LowerProb() const { return Pay().lower_prob; }
-
- const uint8_t *FromBegin() const { return Pay().from; }
- uint8_t *FromBegin() { return Pay().from; }
-
-private:
- struct After {
- // Note that backoff_and_normalize assumes this comes first.
- float prob;
- float lower_prob;
- uint8_t from[];
- };
- const After &Pay() const { return *reinterpret_cast<const After *>(end()); }
- After &Pay() { return *reinterpret_cast<After*>(end()); }
-
- std::size_t backoff_bytes_;
-};
-
-}} // namespaces
-#endif // LM_INTERPOLATE_MERGE_PROBABILITIES_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/merge_test/test1
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/merge_test/test1 b/ext/kenlm/lm/interpolate/merge_test/test1
deleted file mode 100644
index 08d114d..0000000
Binary files a/ext/kenlm/lm/interpolate/merge_test/test1 and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/merge_test/test2
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/merge_test/test2 b/ext/kenlm/lm/interpolate/merge_test/test2
deleted file mode 100644
index fd3a380..0000000
Binary files a/ext/kenlm/lm/interpolate/merge_test/test2 and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/merge_test/test3
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/merge_test/test3 b/ext/kenlm/lm/interpolate/merge_test/test3
deleted file mode 100644
index 6c89d7f..0000000
Binary files a/ext/kenlm/lm/interpolate/merge_test/test3 and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/merge_test/test_bad_order
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/merge_test/test_bad_order b/ext/kenlm/lm/interpolate/merge_test/test_bad_order
deleted file mode 100644
index d50a5e8..0000000
Binary files a/ext/kenlm/lm/interpolate/merge_test/test_bad_order and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/merge_test/test_no_unk
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/merge_test/test_no_unk b/ext/kenlm/lm/interpolate/merge_test/test_no_unk
deleted file mode 100644
index fbcf12d..0000000
--- a/ext/kenlm/lm/interpolate/merge_test/test_no_unk
+++ /dev/null
@@ -1 +0,0 @@
-toto
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/merge_vocab.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/merge_vocab.cc b/ext/kenlm/lm/interpolate/merge_vocab.cc
deleted file mode 100644
index 3e84389..0000000
--- a/ext/kenlm/lm/interpolate/merge_vocab.cc
+++ /dev/null
@@ -1,131 +0,0 @@
-#include "lm/interpolate/merge_vocab.hh"
-
-#include "lm/enumerate_vocab.hh"
-#include "lm/interpolate/universal_vocab.hh"
-#include "lm/lm_exception.hh"
-#include "lm/vocab.hh"
-#include "util/file_piece.hh"
-
-#include <queue>
-#include <string>
-#include <iostream>
-#include <vector>
-
-namespace lm {
-namespace interpolate {
-namespace {
-
-class VocabFileReader {
- public:
- explicit VocabFileReader(const int fd, size_t model_num, uint64_t offset = 0);
-
- VocabFileReader &operator++();
- operator bool() const { return !eof_; }
- uint64_t operator*() const { return Value(); }
-
- uint64_t Value() const { return hash_value_; }
- size_t ModelNum() const { return model_num_; }
- WordIndex CurrentIndex() const { return current_index_; }
-
- StringPiece Word() const { return word_; }
-
- private:
- uint64_t hash_value_;
- WordIndex current_index_;
- bool eof_;
- size_t model_num_;
- StringPiece word_;
- util::FilePiece file_piece_;
-};
-
-VocabFileReader::VocabFileReader(const int fd, const size_t model_num, uint64_t offset) :
- hash_value_(0),
- current_index_(0),
- eof_(false),
- model_num_(model_num),
- file_piece_(fd) {
- word_ = file_piece_.ReadLine('\0');
- UTIL_THROW_IF(word_ != "<unk>",
- FormatLoadException,
- "Vocabulary words are in the wrong place.");
- // setup to initial value
- ++*this;
-}
-
-VocabFileReader &VocabFileReader::operator++() {
- try {
- word_ = file_piece_.ReadLine('\0');
- } catch(util::EndOfFileException &e) {
- eof_ = true;
- return *this;
- }
- uint64_t prev_hash_value = hash_value_;
- hash_value_ = ngram::detail::HashForVocab(word_.data(), word_.size());
-
- // hash values should be monotonically increasing
- UTIL_THROW_IF(hash_value_ < prev_hash_value, FormatLoadException,
- ": word index not monotonically increasing."
- << " model_num: " << model_num_
- << " prev hash: " << prev_hash_value
- << " new hash: " << hash_value_);
-
- ++current_index_;
- return *this;
-}
-
-class CompareFiles {
-public:
- bool operator()(const VocabFileReader* x,
- const VocabFileReader* y)
- { return x->Value() > y->Value(); }
-};
-
-class Readers : public util::FixedArray<VocabFileReader> {
- public:
- Readers(std::size_t number) : util::FixedArray<VocabFileReader>(number) {}
- void push_back(int fd, std::size_t i) {
- new(end()) VocabFileReader(fd, i);
- Constructed();
- }
-};
-
-} // namespace
-
-WordIndex MergeVocab(util::FixedArray<util::scoped_fd> &files, UniversalVocab &vocab, EnumerateVocab &enumerate) {
- typedef std::priority_queue<VocabFileReader*, std::vector<VocabFileReader*>, CompareFiles> HeapType;
- HeapType heap;
- Readers readers(files.size());
- for (size_t i = 0; i < files.size(); ++i) {
- readers.push_back(files[i].release(), i);
- heap.push(&readers.back());
- // initialize first index to 0 for <unk>
- vocab.InsertUniversalIdx(i, 0, 0);
- }
-
- uint64_t prev_hash_value = 0;
- // global_index starts with <unk> which is 0
- WordIndex global_index = 0;
-
- enumerate.Add(0, "<unk>");
- while (!heap.empty()) {
- VocabFileReader* top_vocab_file = heap.top();
- if (top_vocab_file->Value() != prev_hash_value) {
- enumerate.Add(++global_index, top_vocab_file->Word());
- }
- vocab.InsertUniversalIdx(top_vocab_file->ModelNum(),
- top_vocab_file->CurrentIndex(),
- global_index);
-
- prev_hash_value = top_vocab_file->Value();
-
- heap.pop();
- if (++(*top_vocab_file)) {
- heap.push(top_vocab_file);
- }
- }
- return global_index + 1;
-}
-
-} // namespace interpolate
-} // namespace lm
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/merge_vocab.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/merge_vocab.hh b/ext/kenlm/lm/interpolate/merge_vocab.hh
deleted file mode 100644
index cc74d33..0000000
--- a/ext/kenlm/lm/interpolate/merge_vocab.hh
+++ /dev/null
@@ -1,23 +0,0 @@
-#ifndef LM_INTERPOLATE_MERGE_VOCAB_H
-#define LM_INTERPOLATE_MERGE_VOCAB_H
-
-#include "lm/word_index.hh"
-#include "util/file.hh"
-#include "util/fixed_array.hh"
-
-namespace lm {
-
-class EnumerateVocab;
-
-namespace interpolate {
-
-class UniversalVocab;
-
-// Takes ownership of vocab_files.
-// The combined vocabulary is enumerated with enumerate.
-// Returns the size of the combined vocabulary.
-WordIndex MergeVocab(util::FixedArray<util::scoped_fd> &vocab_files, UniversalVocab &vocab, EnumerateVocab &enumerate);
-
-}} // namespaces
-
-#endif // LM_INTERPOLATE_MERGE_VOCAB_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/merge_vocab_test.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/merge_vocab_test.cc b/ext/kenlm/lm/interpolate/merge_vocab_test.cc
deleted file mode 100644
index 6df25b2..0000000
--- a/ext/kenlm/lm/interpolate/merge_vocab_test.cc
+++ /dev/null
@@ -1,126 +0,0 @@
-#define BOOST_TEST_MODULE InterpolateMergeVocabTest
-#include <boost/test/unit_test.hpp>
-
-#include "lm/enumerate_vocab.hh"
-#include "lm/interpolate/merge_vocab.hh"
-#include "lm/interpolate/universal_vocab.hh"
-#include "lm/lm_exception.hh"
-#include "lm/vocab.hh"
-#include "lm/word_index.hh"
-#include "util/file.hh"
-#include "util/file_piece.hh"
-
-#include <cstring>
-
-namespace lm {
-namespace interpolate {
-namespace {
-
-// Stupid bjam permutes the command line arguments randomly.
-class TestFiles {
- public:
- TestFiles() {
- char **argv = boost::unit_test::framework::master_test_suite().argv;
- int argc = boost::unit_test::framework::master_test_suite().argc;
- BOOST_REQUIRE_EQUAL(6, argc);
- for (int i = 1; i < argc; ++i) {
- EndsWithAssign(argv[i], "test1", test[0]);
- EndsWithAssign(argv[i], "test2", test[1]);
- EndsWithAssign(argv[i], "test3", test[2]);
- EndsWithAssign(argv[i], "no_unk", no_unk);
- EndsWithAssign(argv[i], "bad_order", bad_order);
- }
- }
-
- void EndsWithAssign(char *arg, StringPiece value, util::scoped_fd &to) {
- StringPiece str(arg);
- if (str.size() < value.size()) return;
- if (std::memcmp(str.data() + str.size() - value.size(), value.data(), value.size())) return;
- to.reset(util::OpenReadOrThrow(arg));
- }
-
- util::scoped_fd test[3], no_unk, bad_order;
-};
-
-class DoNothingEnumerate : public EnumerateVocab {
- public:
- void Add(WordIndex, const StringPiece &) {}
-};
-
-BOOST_AUTO_TEST_CASE(MergeVocabTest) {
- TestFiles files;
-
- util::FixedArray<util::scoped_fd> used_files(3);
- used_files.push_back(files.test[0].release());
- used_files.push_back(files.test[1].release());
- used_files.push_back(files.test[2].release());
-
- std::vector<lm::WordIndex> model_max_idx;
- model_max_idx.push_back(10);
- model_max_idx.push_back(10);
- model_max_idx.push_back(10);
-
- util::scoped_fd combined(util::MakeTemp("temporary"));
-
- UniversalVocab universal_vocab(model_max_idx);
- {
- ngram::ImmediateWriteWordsWrapper writer(NULL, combined.get(), 0);
- MergeVocab(used_files, universal_vocab, writer);
- }
-
- BOOST_CHECK_EQUAL(universal_vocab.GetUniversalIdx(0, 0), 0);
- BOOST_CHECK_EQUAL(universal_vocab.GetUniversalIdx(1, 0), 0);
- BOOST_CHECK_EQUAL(universal_vocab.GetUniversalIdx(2, 0), 0);
- BOOST_CHECK_EQUAL(universal_vocab.GetUniversalIdx(0, 1), 1);
- BOOST_CHECK_EQUAL(universal_vocab.GetUniversalIdx(1, 1), 2);
- BOOST_CHECK_EQUAL(universal_vocab.GetUniversalIdx(2, 1), 8);
- BOOST_CHECK_EQUAL(universal_vocab.GetUniversalIdx(0, 5), 11);
- BOOST_CHECK_EQUAL(universal_vocab.GetUniversalIdx(1, 3), 4);
- BOOST_CHECK_EQUAL(universal_vocab.GetUniversalIdx(2, 3), 10);
-
- util::FilePiece f(combined.release());
- BOOST_CHECK_EQUAL("<unk>", f.ReadLine('\0'));
- BOOST_CHECK_EQUAL("a", f.ReadLine('\0'));
- BOOST_CHECK_EQUAL("is this", f.ReadLine('\0'));
- BOOST_CHECK_EQUAL("this a", f.ReadLine('\0'));
- BOOST_CHECK_EQUAL("first cut", f.ReadLine('\0'));
- BOOST_CHECK_EQUAL("this", f.ReadLine('\0'));
- BOOST_CHECK_EQUAL("a first", f.ReadLine('\0'));
- BOOST_CHECK_EQUAL("cut", f.ReadLine('\0'));
- BOOST_CHECK_EQUAL("is", f.ReadLine('\0'));
- BOOST_CHECK_EQUAL("i", f.ReadLine('\0'));
- BOOST_CHECK_EQUAL("secd", f.ReadLine('\0'));
- BOOST_CHECK_EQUAL("first", f.ReadLine('\0'));
- BOOST_CHECK_THROW(f.ReadLine('\0'), util::EndOfFileException);
-}
-
-BOOST_AUTO_TEST_CASE(MergeVocabNoUnkTest) {
- TestFiles files;
- util::FixedArray<util::scoped_fd> used_files(1);
- used_files.push_back(files.no_unk.release());
-
- std::vector<lm::WordIndex> model_max_idx;
- model_max_idx.push_back(10);
-
- UniversalVocab universal_vocab(model_max_idx);
- DoNothingEnumerate nothing;
- BOOST_CHECK_THROW(MergeVocab(used_files, universal_vocab, nothing), FormatLoadException);
-}
-
-BOOST_AUTO_TEST_CASE(MergeVocabWrongOrderTest) {
- TestFiles files;
-
- util::FixedArray<util::scoped_fd> used_files(2);
- used_files.push_back(files.test[0].release());
- used_files.push_back(files.bad_order.release());
-
- std::vector<lm::WordIndex> model_max_idx;
- model_max_idx.push_back(10);
- model_max_idx.push_back(10);
-
- lm::interpolate::UniversalVocab universal_vocab(model_max_idx);
- DoNothingEnumerate nothing;
- BOOST_CHECK_THROW(MergeVocab(used_files, universal_vocab, nothing), FormatLoadException);
-}
-
-}}} // namespaces
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/normalize.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/normalize.cc b/ext/kenlm/lm/interpolate/normalize.cc
deleted file mode 100644
index f683f10..0000000
--- a/ext/kenlm/lm/interpolate/normalize.cc
+++ /dev/null
@@ -1,384 +0,0 @@
-#include "lm/interpolate/normalize.hh"
-
-#include "lm/common/compare.hh"
-#include "lm/common/ngram_stream.hh"
-#include "lm/interpolate/backoff_matrix.hh"
-#include "lm/interpolate/bounded_sequence_encoding.hh"
-#include "lm/interpolate/interpolate_info.hh"
-#include "lm/interpolate/merge_probabilities.hh"
-#include "lm/weights.hh"
-#include "lm/word_index.hh"
-#include "util/fixed_array.hh"
-#include "util/scoped.hh"
-#include "util/stream/stream.hh"
-#include "util/stream/rewindable_stream.hh"
-
-#include <functional>
-#include <queue>
-#include <vector>
-
-namespace lm { namespace interpolate {
-namespace {
-
-struct SuffixLexicographicLess : public std::binary_function<NGramHeader, NGramHeader, bool> {
- bool operator()(const NGramHeader first, const NGramHeader second) const {
- for (const WordIndex *f = first.end() - 1, *s = second.end() - 1; f >= first.begin() && s >= second.begin(); --f, --s) {
- if (*f < *s) return true;
- if (*f > *s) return false;
- }
- return first.size() < second.size();
- }
-};
-
-class BackoffQueueEntry {
- public:
- BackoffQueueEntry(float &entry, const util::stream::ChainPosition &position)
- : entry_(entry), stream_(position) {
- entry_ = 0.0;
- }
-
- operator bool() const { return stream_; }
-
- NGramHeader operator*() const { return *stream_; }
- const NGramHeader *operator->() const { return &*stream_; }
-
- void Enter() {
- entry_ = stream_->Value().backoff;
- }
-
- BackoffQueueEntry &Next() {
- entry_ = 0.0;
- ++stream_;
- return *this;
- }
-
- private:
- float &entry_;
- NGramStream<ProbBackoff> stream_;
-};
-
-struct PtrGreater : public std::binary_function<const BackoffQueueEntry *, const BackoffQueueEntry *, bool> {
- bool operator()(const BackoffQueueEntry *first, const BackoffQueueEntry *second) const {
- return SuffixLexicographicLess()(**second, **first);
- }
-};
-
-class EntryOwner : public util::FixedArray<BackoffQueueEntry> {
- public:
- void push_back(float &entry, const util::stream::ChainPosition &position) {
- new (end()) BackoffQueueEntry(entry, position);
- Constructed();
- }
-};
-
-std::size_t MaxOrder(const util::FixedArray<util::stream::ChainPositions> &model) {
- std::size_t ret = 0;
- for (const util::stream::ChainPositions *m = model.begin(); m != model.end(); ++m) {
- ret = std::max(ret, m->size());
- }
- return ret;
-}
-
-class BackoffManager {
- public:
- explicit BackoffManager(const util::FixedArray<util::stream::ChainPositions> &models)
- : entered_(MaxOrder(models)), matrix_(models.size(), MaxOrder(models)), skip_write_(MaxOrder(models)) {
- std::size_t total = 0;
- for (const util::stream::ChainPositions *m = models.begin(); m != models.end(); ++m) {
- total += m->size();
- }
- for (std::size_t i = 0; i < MaxOrder(models); ++i) {
- entered_.push_back(models.size());
- }
- owner_.Init(total);
- for (const util::stream::ChainPositions *m = models.begin(); m != models.end(); ++m) {
- for (const util::stream::ChainPosition *j = m->begin(); j != m->end(); ++j) {
- owner_.push_back(matrix_.Backoff(m - models.begin(), j - m->begin()), *j);
- if (owner_.back()) {
- queue_.push(&owner_.back());
- }
- }
- }
- }
-
- void SetupSkip(std::size_t order, util::stream::Stream &stream) {
- skip_write_[order - 2] = &stream;
- }
-
- // Move up the backoffs for the given n-gram. The n-grams must be provided
- // in suffix lexicographic order.
- void Enter(const NGramHeader &to) {
- // Check that we exited properly.
- for (std::size_t i = to.Order() - 1; i < entered_.size(); ++i) {
- assert(entered_[i].empty());
- }
- SuffixLexicographicLess less;
- while (!queue_.empty() && less(**queue_.top(), to))
- SkipRecord();
- while (TopMatches(to)) {
- BackoffQueueEntry *matches = queue_.top();
- entered_[to.Order() - 1].push_back(matches);
- matches->Enter();
- queue_.pop();
- }
- }
-
- void Exit(std::size_t order_minus_1) {
- for (BackoffQueueEntry **i = entered_[order_minus_1].begin(); i != entered_[order_minus_1].end(); ++i) {
- if ((*i)->Next())
- queue_.push(*i);
- }
- entered_[order_minus_1].clear();
- }
-
- float Get(std::size_t model, std::size_t order_minus_1) const {
- return matrix_.Backoff(model, order_minus_1);
- }
-
- void Finish() {
- while (!queue_.empty())
- SkipRecord();
- }
-
- private:
- void SkipRecord() {
- BackoffQueueEntry *top = queue_.top();
- queue_.pop();
- // Is this the last instance of the n-gram?
- if (!TopMatches(**top)) {
- // An n-gram is being skipped. Called once per skipped n-gram,
- // regardless of how many models it comes from.
- *reinterpret_cast<float*>(skip_write_[(*top)->Order() - 1]->Get()) = 0.0;
- ++*skip_write_[(*top)->Order() - 1];
- }
- if (top->Next())
- queue_.push(top);
- }
-
- bool TopMatches(const NGramHeader &header) const {
- return !queue_.empty() && (*queue_.top())->Order() == header.Order() && std::equal(header.begin(), header.end(), (*queue_.top())->begin());
- }
-
- EntryOwner owner_;
- std::priority_queue<BackoffQueueEntry*, std::vector<BackoffQueueEntry*>, PtrGreater> queue_;
-
- // Indexed by order then just all the matching models.
- util::FixedArray<util::FixedArray<BackoffQueueEntry*> > entered_;
-
- std::size_t order_;
-
- BackoffMatrix matrix_;
-
- std::vector<util::stream::Stream*> skip_write_;
-};
-
-typedef long double Accum;
-
-// Handles n-grams of the same order, using recursion to call another instance
-// for higher orders.
-class Recurse {
- public:
- Recurse(
- const InterpolateInfo &info, // Must stay alive the entire time.
- std::size_t order,
- const util::stream::ChainPosition &merged_probs,
- const util::stream::ChainPosition &prob_out,
- const util::stream::ChainPosition &backoff_out,
- BackoffManager &backoffs,
- Recurse *higher) // higher is null for the highest order.
- : order_(order),
- encoding_(MakeEncoder(info, order)),
- input_(merged_probs, PartialProbGamma(order, encoding_.EncodedLength())),
- prob_out_(prob_out),
- backoff_out_(backoff_out),
- backoffs_(backoffs),
- lambdas_(&*info.lambdas.begin()),
- higher_(higher),
- decoded_backoffs_(info.Models()),
- extended_context_(order - 1) {
- // This is only for bigrams and above. Summing unigrams is a much easier case.
- assert(order >= 2);
- }
-
- // context = w_1^{n-1}
- // z_lower = Z(w_2^{n-1})
- // Input:
- // Merged probabilities without backoff applied in input_.
- // Backoffs via backoffs_.
- // Calculates:
- // Z(w_1^{n-1}): intermediate only.
- // p_I(x | w_1^{n-1}) for all x: w_1^{n-1}x exists: Written to prob_out_.
- // b_I(w_1^{n-1}): Written to backoff_out_.
- void SameContext(const NGramHeader &context, Accum z_lower) {
- assert(context.size() == order_ - 1);
- backoffs_.Enter(context);
- prob_out_.Mark();
-
- // This is the backoff term that applies when one assumes everything backs off:
- // \prod_i b_i(w_1^{n-1})^{\lambda_i}.
- Accum backoff_once = 0.0;
- for (std::size_t m = 0; m < decoded_backoffs_.size(); ++m) {
- backoff_once += lambdas_[m] * backoffs_.Get(m, order_ - 2);
- }
-
- Accum z_delta = 0.0;
- std::size_t count = 0;
- for (; input_ && std::equal(context.begin(), context.end(), input_->begin()); ++input_, ++prob_out_, ++count) {
- // Apply backoffs to probabilities.
- // TODO: change bounded sequence encoding to have an iterator for decoding instead of doing a copy here.
- encoding_.Decode(input_->FromBegin(), &*decoded_backoffs_.begin());
- for (std::size_t m = 0; m < NumModels(); ++m) {
- // Apply the backoffs as instructed for model m.
- float accumulated = 0.0;
- // Change backoffs for [order it backed off to, order - 1) except
- // with 0-indexing. There is still the potential to charge backoff
- // for order - 1, which is done later. The backoffs charged here
- // are b_m(w_{n-1}^{n-1}) ... b_m(w_2^{n-1})
- for (unsigned char backed_to = decoded_backoffs_[m]; backed_to < order_ - 2; ++backed_to) {
- accumulated += backoffs_.Get(m, backed_to);
- }
- float lambda = lambdas_[m];
- // Lower p(x | w_2^{n-1}) gets all the backoffs except the highest.
- input_->LowerProb() += accumulated * lambda;
- // Charge the backoff b(w_1^{n-1}) if applicable, but only to attain p(x | w_1^{n-1})
- if (decoded_backoffs_[m] < order_ - 1) {
- accumulated += backoffs_.Get(m, order_ - 2);
- }
- input_->Prob() += accumulated * lambda;
- }
- // TODO: better precision/less operations here.
- z_delta += pow(10.0, input_->Prob()) - pow(10.0, input_->LowerProb() + backoff_once);
-
- // Write unnormalized probability record.
- std::copy(input_->begin(), input_->end(), reinterpret_cast<WordIndex*>(prob_out_.Get()));
- ProbWrite() = input_->Prob();
- }
- // TODO numerical precision.
- Accum z = log10(pow(10.0, z_lower + backoff_once) + z_delta);
-
- // Normalize.
- prob_out_.Rewind();
- for (std::size_t i = 0; i < count; ++i, ++prob_out_) {
- ProbWrite() -= z;
- }
- // This allows the stream to release data.
- prob_out_.Mark();
-
- // Output backoff.
- *reinterpret_cast<float*>(backoff_out_.Get()) = z_lower + backoff_once - z;
- ++backoff_out_;
-
- if (higher_.get())
- higher_->ExtendContext(context, z);
-
- backoffs_.Exit(order_ - 2);
- }
-
- // Call is given a context and z(context).
- // Evaluates y context x for all y,x.
- void ExtendContext(const NGramHeader &middle, Accum z_lower) {
- assert(middle.size() == order_ - 2);
- // Copy because the input will advance. TODO avoid this copy by sharing amongst classes.
- std::copy(middle.begin(), middle.end(), extended_context_.begin() + 1);
- while (input_ && std::equal(middle.begin(), middle.end(), input_->begin() + 1)) {
- *extended_context_.begin() = *input_->begin();
- SameContext(NGramHeader(&*extended_context_.begin(), order_ - 1), z_lower);
- }
- }
-
- void Finish() {
- assert(!input_);
- prob_out_.Poison();
- backoff_out_.Poison();
- if (higher_.get())
- higher_->Finish();
- }
-
- // The BackoffManager class also injects backoffs when it skips ahead e.g. b(</s>) = 1
- util::stream::Stream &BackoffStream() { return backoff_out_; }
-
- private:
- // Write the probability to the correct place in prob_out_. Should use a proxy but currently incompatible with RewindableStream.
- float &ProbWrite() {
- return *reinterpret_cast<float*>(reinterpret_cast<uint8_t*>(prob_out_.Get()) + order_ * sizeof(WordIndex));
- }
-
- std::size_t NumModels() const { return decoded_backoffs_.size(); }
-
- const std::size_t order_;
-
- const BoundedSequenceEncoding encoding_;
-
- ProxyStream<PartialProbGamma> input_;
- util::stream::RewindableStream prob_out_;
- util::stream::Stream backoff_out_;
-
- BackoffManager &backoffs_;
- const float *const lambdas_;
-
- // Higher order instance of this same class.
- util::scoped_ptr<Recurse> higher_;
-
- // Temporary in SameContext.
- std::vector<unsigned char> decoded_backoffs_;
- // Temporary in ExtendContext.
- std::vector<WordIndex> extended_context_;
-};
-
-class Thread {
- public:
- Thread(const InterpolateInfo &info, util::FixedArray<util::stream::ChainPositions> &models_by_order, util::stream::Chains &prob_out, util::stream::Chains &backoff_out)
- : info_(info), models_by_order_(models_by_order), prob_out_(prob_out), backoff_out_(backoff_out) {}
-
- void Run(const util::stream::ChainPositions &merged_probabilities) {
- // Unigrams do not have enocded backoff info.
- ProxyStream<PartialProbGamma> in(merged_probabilities[0], PartialProbGamma(1, 0));
- util::stream::RewindableStream prob_write(prob_out_[0]);
- Accum z = 0.0;
- prob_write.Mark();
- WordIndex count = 0;
- for (; in; ++in, ++prob_write, ++count) {
- // Note assumption that probabilitity comes first
- memcpy(prob_write.Get(), in.Get(), sizeof(WordIndex) + sizeof(float));
- z += pow(10.0, in->Prob());
- }
- // TODO HACK TODO: lmplz outputs p(<s>) = 1 to get q to compute nicely. That will always result in 1.0 more than it should be.
- z -= 1.0;
- float log_z = log10(z);
- prob_write.Rewind();
- // Normalize unigram probabilities.
- for (WordIndex i = 0; i < count; ++i, ++prob_write) {
- *reinterpret_cast<float*>(reinterpret_cast<uint8_t*>(prob_write.Get()) + sizeof(WordIndex)) -= log_z;
- }
- prob_write.Poison();
-
- // Now setup the higher orders.
- util::scoped_ptr<Recurse> higher_order;
- BackoffManager backoffs(models_by_order_);
- std::size_t max_order = merged_probabilities.size();
- for (std::size_t order = max_order; order >= 2; --order) {
- higher_order.reset(new Recurse(info_, order, merged_probabilities[order - 1], prob_out_[order - 1], backoff_out_[order - 2], backoffs, higher_order.release()));
- backoffs.SetupSkip(order, higher_order->BackoffStream());
- }
- if (max_order > 1) {
- higher_order->ExtendContext(NGramHeader(NULL, 0), log_z);
- higher_order->Finish();
- }
- }
-
- private:
- const InterpolateInfo info_;
- util::FixedArray<util::stream::ChainPositions> &models_by_order_;
- util::stream::ChainPositions prob_out_;
- util::stream::ChainPositions backoff_out_;
-};
-
-} // namespace
-
-void Normalize(const InterpolateInfo &info, util::FixedArray<util::stream::ChainPositions> &models_by_order, util::stream::Chains &merged_probabilities, util::stream::Chains &prob_out, util::stream::Chains &backoff_out) {
- assert(prob_out.size() == backoff_out.size() + 1);
- // Arbitrarily put the thread on the merged_probabilities Chains.
- merged_probabilities >> Thread(info, models_by_order, prob_out, backoff_out);
-}
-
-}} // namespaces
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/normalize.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/normalize.hh b/ext/kenlm/lm/interpolate/normalize.hh
deleted file mode 100644
index dd26b25..0000000
--- a/ext/kenlm/lm/interpolate/normalize.hh
+++ /dev/null
@@ -1,35 +0,0 @@
-#ifndef LM_INTERPOLATE_NORMALIZE_H
-#define LM_INTERPOLATE_NORMALIZE_H
-
-#include "util/fixed_array.hh"
-
-/* Pass 2:
- * - Multiply backoff weights by the backed off probabilities from pass 1.
- * - Compute the normalization factor Z.
- * - Send Z to the next highest order.
- * - Rewind and divide by Z.
- */
-
-namespace util { namespace stream {
-class ChainPositions;
-class Chains;
-}} // namespaces
-
-namespace lm { namespace interpolate {
-
-struct InterpolateInfo;
-
-void Normalize(
- const InterpolateInfo &info,
- // Input full models for backoffs. Assumes that renumbering has been done. Suffix order.
- util::FixedArray<util::stream::ChainPositions> &models_by_order,
- // Input PartialProbGamma from MergeProbabilities. Context order.
- util::stream::Chains &merged_probabilities,
- // Output NGram<float> with normalized probabilities. Context order.
- util::stream::Chains &probabilities_out,
- // Output bare floats with backoffs. Note backoffs.size() == order - 1. Suffix order.
- util::stream::Chains &backoffs_out);
-
-}} // namespaces
-
-#endif // LM_INTERPOLATE_NORMALIZE_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/normalize_test.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/normalize_test.cc b/ext/kenlm/lm/interpolate/normalize_test.cc
deleted file mode 100644
index fe220f3..0000000
--- a/ext/kenlm/lm/interpolate/normalize_test.cc
+++ /dev/null
@@ -1,86 +0,0 @@
-#include "lm/interpolate/normalize.hh"
-
-#include "lm/interpolate/interpolate_info.hh"
-#include "lm/interpolate/merge_probabilities.hh"
-#include "lm/common/ngram_stream.hh"
-#include "util/stream/chain.hh"
-#include "util/stream/multi_stream.hh"
-
-#define BOOST_TEST_MODULE NormalizeTest
-#include <boost/test/unit_test.hpp>
-
-namespace lm { namespace interpolate { namespace {
-
-// log without backoff
-const float kInputs[] = {-0.3, 1.2, -9.8, 4.0, -7.0, 0.0};
-
-class WriteInput {
- public:
- WriteInput() {}
- void Run(const util::stream::ChainPosition &to) {
- util::stream::Stream out(to);
- for (WordIndex i = 0; i < sizeof(kInputs) / sizeof(float); ++i, ++out) {
- memcpy(out.Get(), &i, sizeof(WordIndex));
- memcpy((uint8_t*)out.Get() + sizeof(WordIndex), &kInputs[i], sizeof(float));
- }
- out.Poison();
- }
-};
-
-void CheckOutput(const util::stream::ChainPosition &from) {
- NGramStream<float> in(from);
- float sum = 0.0;
- for (WordIndex i = 0; i < sizeof(kInputs) / sizeof(float) - 1 /* <s> at the end */; ++i) {
- sum += pow(10.0, kInputs[i]);
- }
- sum = log10(sum);
- BOOST_REQUIRE(in);
- BOOST_CHECK_CLOSE(kInputs[0] - sum, in->Value(), 0.0001);
- BOOST_REQUIRE(++in);
- BOOST_CHECK_CLOSE(kInputs[1] - sum, in->Value(), 0.0001);
- BOOST_REQUIRE(++in);
- BOOST_CHECK_CLOSE(kInputs[2] - sum, in->Value(), 0.0001);
- BOOST_REQUIRE(++in);
- BOOST_CHECK_CLOSE(kInputs[3] - sum, in->Value(), 0.0001);
- BOOST_REQUIRE(++in);
- BOOST_CHECK_CLOSE(kInputs[4] - sum, in->Value(), 0.0001);
- BOOST_REQUIRE(++in);
- BOOST_CHECK_CLOSE(kInputs[5] - sum, in->Value(), 0.0001);
- BOOST_CHECK(!++in);
-}
-
-BOOST_AUTO_TEST_CASE(Unigrams) {
- InterpolateInfo info;
- info.lambdas.push_back(2.0);
- info.lambdas.push_back(-0.1);
- info.orders.push_back(1);
- info.orders.push_back(1);
-
- BOOST_CHECK_EQUAL(0, MakeEncoder(info, 1).EncodedLength());
-
- // No backoffs.
- util::stream::Chains blank(0);
- util::FixedArray<util::stream::ChainPositions> models_by_order(2);
- models_by_order.push_back(blank);
- models_by_order.push_back(blank);
-
- util::stream::Chains merged_probabilities(1);
- util::stream::Chains probabilities_out(1);
- util::stream::Chains backoffs_out(0);
-
- merged_probabilities.push_back(util::stream::ChainConfig(sizeof(WordIndex) + sizeof(float) + sizeof(float), 2, 24));
- probabilities_out.push_back(util::stream::ChainConfig(sizeof(WordIndex) + sizeof(float), 2, 100));
-
- merged_probabilities[0] >> WriteInput();
- Normalize(info, models_by_order, merged_probabilities, probabilities_out, backoffs_out);
-
- util::stream::ChainPosition checker(probabilities_out[0].Add());
-
- merged_probabilities >> util::stream::kRecycle;
- probabilities_out >> util::stream::kRecycle;
-
- CheckOutput(checker);
- probabilities_out.Wait();
-}
-
-}}} // namespaces
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/perf_enum_gv_main.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/perf_enum_gv_main.cc b/ext/kenlm/lm/interpolate/perf_enum_gv_main.cc
deleted file mode 100644
index a68e9e9..0000000
--- a/ext/kenlm/lm/interpolate/perf_enum_gv_main.cc
+++ /dev/null
@@ -1,215 +0,0 @@
-/*
-Usage example
-1) Download from http://www.gwinnup.org/lminterp/train-params-output.tar.bz2
-2) then run perf_enum_gv -t lm.en.dev -m model-a.3.srilm -m model-b.3.srilm -m model-c.3.srilm
- */
-
-#include "lm/ngram_query.hh"
-#include "lm/model.hh"
-#include "lm/word_index.hh"
-#include "lm/interpolate/enumerate_global_vocab.hh"
-
-#include "util/fixed_array.hh"
-#include "util/usage.hh"
-
-#include <string>
-#include <vector>
-#include <iostream>
-#include <fstream>
-#include <map>
-
-#include <boost/program_options.hpp>
-#include <boost/version.hpp>
-#include <boost/foreach.hpp>
-
-#include <Eigen/Eigen>
-
-#include <iostream>
-#include <sys/time.h>
-
-inline double deltaTV(const timeval& s, const timeval& e)
-{
- return (e.tv_sec - s.tv_sec)*1000.0 + (e.tv_usec - s.tv_usec)/1000.0;
-}
-
-typedef struct timeval Wall;
-Wall GetWall() {
- struct timeval tv;
- gettimeofday(&tv, NULL);
- return tv;
-}
-
-typedef Eigen::MatrixXf FMatrix;
-typedef Eigen::VectorXf FVector;
-
-
-bool HAS_BIAS = true;
-
-using namespace lm::ngram;
-using namespace lm;
-
-inline void logProb(Model * model, const std::vector<std::string>& ctx, const std::string& word) {
-
- // Horribly inefficient
- const Vocabulary &vocab = model->GetVocabulary();
-
- State nextState; //throwaway
-
- WordIndex word_idx = vocab.Index(word);
- WordIndex context_idx[ctx.size()];
-
- //reverse context
- for(unsigned int i = 0; i < ctx.size(); i++) {
- context_idx[ctx.size() - 1 - i] = vocab.Index(ctx[i]);
- }
- FullScoreReturn score = model->FullScoreForgotState(context_idx, &(context_idx[ctx.size() -1]), word_idx, nextState);
-}
-
-void set_features(const std::vector<std::string>& ctx,
- const std::string& word,
- const std::vector<Model *>& models,
- FVector& v) {
-
- for (unsigned i=0; i < models.size(); ++i)
- logProb(models[i], ctx, word);
-
-}
-
-//const util::FixedArray<Model *>& models)
-void train_params(
- const std::vector<std::vector<std::string> >& corpus,
- const std::vector<std::string>& vocab,
- const std::vector<Model *>& models) {
- using namespace std;
-
- vector<string> context(5, "<s>");
- const int ITERATIONS = 10;
- const int nlambdas = models.size(); // #models
- FVector params = FVector::Zero(nlambdas);
- vector<FVector> feats(vocab.size(), params);
- static Wall start,stop;
-
- for (int iter = 0; iter < ITERATIONS; ++iter) { // iterations
- std::cout << "iteration: " << iter
- << " corpus size " << corpus.size()
- << std::endl;
- for (unsigned ci = 0; ci < corpus.size(); ++ci) { // sentences in tuning corpus
- const vector<string>& sentence = corpus[ci];
- context.resize(5);
- for (unsigned t = 0; t < sentence.size(); ++t) { // words in sentence
- std::cout << "sentence " << ci << " word " << t << std::endl;
- start = GetWall();
- const string& ref_word_string = sentence[t];
- for (unsigned i = 0; i < vocab.size(); ++i) { // vocab
- set_features(context, vocab[i], models, feats[i]);
- }
- stop = GetWall();
- std::cout << " time elapsed = " << deltaTV(start,stop) << std::endl;
- context.push_back(ref_word_string);
- }
- }
- }
-}
-
-int main(int argc, char** argv) {
-
- std::string tuning_data;
- std::vector<std::string> lms;
-
- try {
- namespace po = boost::program_options;
- po::options_description options("train-params");
-
- options.add_options()
- ("help,h", po::bool_switch(), "Show this help message")
- ("no_bias_term,B", po::bool_switch(), "Do not include a 'bias' feature")
- ("tuning_data,t", po::value<std::string>(&tuning_data), "File to tune perplexity on")
- ("model,m", po::value<std::vector<std::string> >(&lms), "Language models in KenLM format to interpolate");
- po::variables_map vm;
- po::store(po::parse_command_line(argc, argv, options), vm);
-
- // Display help
- if(argc == 1 || vm["help"].as<bool>()) {
- std::cerr << options << std::endl;
- return 1;
- }
- if (vm["no_bias_term"].as<bool>())
- HAS_BIAS = false;
- lms = vm["model"].as<std::vector<std::string> >();
- tuning_data = vm["tuning_data"].as<std::string>();
- }
- catch(const std::exception &e) {
-
- std::cerr << e.what() << std::endl;
- return 1;
-
- }
- if (lms.size() < 2) {
- std::cerr << "Please specify at least two language model files with -m LM.KLM\n";
- return 1;
- }
- if (tuning_data.empty()) {
- std::cerr << "Please specify tuning set with -t FILE.TXT\n";
- return 1;
- }
-
- std::map<std::string, int*> vmap;
- util::FixedArray<WordIndex> vm(2);
-
- //stuff it into the
- EnumerateGlobalVocab * globalVocabBuilder = new EnumerateGlobalVocab(&vmap, lms.size());
- // EnumerateGlobalVocab * globalVocabBuilder = new EnumerateGlobalVocab(vm);
-
- Config cfg;
- cfg.enumerate_vocab = (EnumerateVocab *) globalVocabBuilder;
-
- //load models
- //util::FixedArray<Model *> models(lms.size());
- std::vector<Model *> models;
- for(int i=0; i < lms.size(); i++) {
- std::cerr << "Loading LM file: " << lms[i] << std::endl;
-
- //haaaack
- globalVocabBuilder->SetCurModel(i); //yes this is dumb
-
- //models[i] = new Model(lms[i].c_str());
- Model * this_model = new Model(lms[i].c_str(), cfg);
- models.push_back( this_model );
-
- }
-
- //assemble vocabulary vector
- std::vector<std::string> vocab;
- std::cerr << "Global Vocab Map has size: " << vmap.size() << std::endl;
-
- std::pair<StringPiece,int *> me;
-
- for(std::map<std::string, int*>::iterator iter = vmap.begin(); iter != vmap.end(); ++iter) {
- vocab.push_back(iter->first);
- }
- std::cerr << "Vocab vector has size: " << vocab.size() << std::endl;
-
- //load context sorted ngrams into vector of vectors
- std::vector<std::vector<std::string> > corpus;
-
- std::cerr << "Loading context-sorted ngrams: " << tuning_data << std::endl;
- std::ifstream infile(tuning_data);
-
- for(std::string line; std::getline(infile, line); ) {
-
- std::vector<std::string> words; {
-
- std::stringstream stream(line);
- std::string word;
-
- while(stream >> word) {
- words.push_back(word);
- }
- }
- corpus.push_back(words);
- }
-
- train_params(corpus, vocab, models);
-
- return 0;
-}
[15/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/binary_format.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/binary_format.cc b/ext/kenlm/lm/binary_format.cc
deleted file mode 100644
index 802943f..0000000
--- a/ext/kenlm/lm/binary_format.cc
+++ /dev/null
@@ -1,302 +0,0 @@
-#include "lm/binary_format.hh"
-
-#include "lm/lm_exception.hh"
-#include "util/file.hh"
-#include "util/file_piece.hh"
-
-#include <cstddef>
-#include <cstring>
-#include <limits>
-#include <string>
-#include <cstdlib>
-
-#include <stdint.h>
-
-namespace lm {
-namespace ngram {
-
-const char *kModelNames[6] = {"probing hash tables", "probing hash tables with rest costs", "trie", "trie with quantization", "trie with array-compressed pointers", "trie with quantization and array-compressed pointers"};
-
-namespace {
-const char kMagicBeforeVersion[] = "mmap lm http://kheafield.com/code format version";
-const char kMagicBytes[] = "mmap lm http://kheafield.com/code format version 5\n\0";
-// This must be shorter than kMagicBytes and indicates an incomplete binary file (i.e. build failed).
-const char kMagicIncomplete[] = "mmap lm http://kheafield.com/code incomplete\n";
-const long int kMagicVersion = 5;
-
-// Old binary files built on 32-bit machines have this header.
-// TODO: eliminate with next binary release.
-struct OldSanity {
- char magic[sizeof(kMagicBytes)];
- float zero_f, one_f, minus_half_f;
- WordIndex one_word_index, max_word_index;
- uint64_t one_uint64;
-
- void SetToReference() {
- std::memset(this, 0, sizeof(OldSanity));
- std::memcpy(magic, kMagicBytes, sizeof(magic));
- zero_f = 0.0; one_f = 1.0; minus_half_f = -0.5;
- one_word_index = 1;
- max_word_index = std::numeric_limits<WordIndex>::max();
- one_uint64 = 1;
- }
-};
-
-
-// Test values aligned to 8 bytes.
-struct Sanity {
- char magic[ALIGN8(sizeof(kMagicBytes))];
- float zero_f, one_f, minus_half_f;
- WordIndex one_word_index, max_word_index, padding_to_8;
- uint64_t one_uint64;
-
- void SetToReference() {
- std::memset(this, 0, sizeof(Sanity));
- std::memcpy(magic, kMagicBytes, sizeof(kMagicBytes));
- zero_f = 0.0; one_f = 1.0; minus_half_f = -0.5;
- one_word_index = 1;
- max_word_index = std::numeric_limits<WordIndex>::max();
- padding_to_8 = 0;
- one_uint64 = 1;
- }
-};
-
-std::size_t TotalHeaderSize(unsigned char order) {
- return ALIGN8(sizeof(Sanity) + sizeof(FixedWidthParameters) + sizeof(uint64_t) * order);
-}
-
-void WriteHeader(void *to, const Parameters ¶ms) {
- Sanity header = Sanity();
- header.SetToReference();
- std::memcpy(to, &header, sizeof(Sanity));
- char *out = reinterpret_cast<char*>(to) + sizeof(Sanity);
-
- *reinterpret_cast<FixedWidthParameters*>(out) = params.fixed;
- out += sizeof(FixedWidthParameters);
-
- uint64_t *counts = reinterpret_cast<uint64_t*>(out);
- for (std::size_t i = 0; i < params.counts.size(); ++i) {
- counts[i] = params.counts[i];
- }
-}
-
-} // namespace
-
-bool IsBinaryFormat(int fd) {
- const uint64_t size = util::SizeFile(fd);
- if (size == util::kBadSize || (size <= static_cast<uint64_t>(sizeof(Sanity)))) return false;
- // Try reading the header.
- util::scoped_memory memory;
- try {
- util::MapRead(util::LAZY, fd, 0, sizeof(Sanity), memory);
- } catch (const util::Exception &e) {
- return false;
- }
- Sanity reference_header = Sanity();
- reference_header.SetToReference();
- if (!std::memcmp(memory.get(), &reference_header, sizeof(Sanity))) return true;
- if (!std::memcmp(memory.get(), kMagicIncomplete, strlen(kMagicIncomplete))) {
- UTIL_THROW(FormatLoadException, "This binary file did not finish building");
- }
- if (!std::memcmp(memory.get(), kMagicBeforeVersion, strlen(kMagicBeforeVersion))) {
- char *end_ptr;
- const char *begin_version = static_cast<const char*>(memory.get()) + strlen(kMagicBeforeVersion);
- long int version = std::strtol(begin_version, &end_ptr, 10);
- if ((end_ptr != begin_version) && version != kMagicVersion) {
- UTIL_THROW(FormatLoadException, "Binary file has version " << version << " but this implementation expects version " << kMagicVersion << " so you'll have to use the ARPA to rebuild your binary");
- }
-
- OldSanity old_sanity = OldSanity();
- old_sanity.SetToReference();
- UTIL_THROW_IF(!std::memcmp(memory.get(), &old_sanity, sizeof(OldSanity)), FormatLoadException, "Looks like this is an old 32-bit format. The old 32-bit format has been removed so that 64-bit and 32-bit files are exchangeable.");
- UTIL_THROW(FormatLoadException, "File looks like it should be loaded with mmap, but the test values don't match. Try rebuilding the binary format LM using the same code revision, compiler, and architecture");
- }
- return false;
-}
-
-void ReadHeader(int fd, Parameters &out) {
- util::SeekOrThrow(fd, sizeof(Sanity));
- util::ReadOrThrow(fd, &out.fixed, sizeof(out.fixed));
- if (out.fixed.probing_multiplier < 1.0)
- UTIL_THROW(FormatLoadException, "Binary format claims to have a probing multiplier of " << out.fixed.probing_multiplier << " which is < 1.0.");
-
- out.counts.resize(static_cast<std::size_t>(out.fixed.order));
- if (out.fixed.order) util::ReadOrThrow(fd, &*out.counts.begin(), sizeof(uint64_t) * out.fixed.order);
-}
-
-void MatchCheck(ModelType model_type, unsigned int search_version, const Parameters ¶ms) {
- if (params.fixed.model_type != model_type) {
- if (static_cast<unsigned int>(params.fixed.model_type) >= (sizeof(kModelNames) / sizeof(const char *)))
- UTIL_THROW(FormatLoadException, "The binary file claims to be model type " << static_cast<unsigned int>(params.fixed.model_type) << " but this is not implemented for in this inference code.");
- UTIL_THROW(FormatLoadException, "The binary file was built for " << kModelNames[params.fixed.model_type] << " but the inference code is trying to load " << kModelNames[model_type]);
- }
- UTIL_THROW_IF(search_version != params.fixed.search_version, FormatLoadException, "The binary file has " << kModelNames[params.fixed.model_type] << " version " << params.fixed.search_version << " but this code expects " << kModelNames[params.fixed.model_type] << " version " << search_version);
-}
-
-const std::size_t kInvalidSize = static_cast<std::size_t>(-1);
-
-BinaryFormat::BinaryFormat(const Config &config)
- : write_method_(config.write_method), write_mmap_(config.write_mmap), load_method_(config.load_method),
- header_size_(kInvalidSize), vocab_size_(kInvalidSize), vocab_string_offset_(kInvalidOffset) {}
-
-void BinaryFormat::InitializeBinary(int fd, ModelType model_type, unsigned int search_version, Parameters ¶ms) {
- file_.reset(fd);
- write_mmap_ = NULL; // Ignore write requests; this is already in binary format.
- ReadHeader(fd, params);
- MatchCheck(model_type, search_version, params);
- header_size_ = TotalHeaderSize(params.counts.size());
-}
-
-void BinaryFormat::ReadForConfig(void *to, std::size_t amount, uint64_t offset_excluding_header) const {
- assert(header_size_ != kInvalidSize);
- util::ErsatzPRead(file_.get(), to, amount, offset_excluding_header + header_size_);
-}
-
-void *BinaryFormat::LoadBinary(std::size_t size) {
- assert(header_size_ != kInvalidSize);
- const uint64_t file_size = util::SizeFile(file_.get());
- // The header is smaller than a page, so we have to map the whole header as well.
- uint64_t total_map = static_cast<uint64_t>(header_size_) + static_cast<uint64_t>(size);
- UTIL_THROW_IF(file_size != util::kBadSize && file_size < total_map, FormatLoadException, "Binary file has size " << file_size << " but the headers say it should be at least " << total_map);
-
- util::MapRead(load_method_, file_.get(), 0, util::CheckOverflow(total_map), mapping_);
-
- vocab_string_offset_ = total_map;
- return reinterpret_cast<uint8_t*>(mapping_.get()) + header_size_;
-}
-
-void *BinaryFormat::SetupJustVocab(std::size_t memory_size, uint8_t order) {
- vocab_size_ = memory_size;
- if (!write_mmap_) {
- header_size_ = 0;
- util::HugeMalloc(memory_size, true, memory_vocab_);
- return reinterpret_cast<uint8_t*>(memory_vocab_.get());
- }
- header_size_ = TotalHeaderSize(order);
- std::size_t total = util::CheckOverflow(static_cast<uint64_t>(header_size_) + static_cast<uint64_t>(memory_size));
- file_.reset(util::CreateOrThrow(write_mmap_));
- // some gccs complain about uninitialized variables even though all enum values are covered.
- void *vocab_base = NULL;
- switch (write_method_) {
- case Config::WRITE_MMAP:
- mapping_.reset(util::MapZeroedWrite(file_.get(), total), total, util::scoped_memory::MMAP_ALLOCATED);
- util::AdviseHugePages(vocab_base, total);
- vocab_base = mapping_.get();
- break;
- case Config::WRITE_AFTER:
- util::ResizeOrThrow(file_.get(), 0);
- util::HugeMalloc(total, true, memory_vocab_);
- vocab_base = memory_vocab_.get();
- break;
- }
- strncpy(reinterpret_cast<char*>(vocab_base), kMagicIncomplete, header_size_);
- return reinterpret_cast<uint8_t*>(vocab_base) + header_size_;
-}
-
-void *BinaryFormat::GrowForSearch(std::size_t memory_size, std::size_t vocab_pad, void *&vocab_base) {
- assert(vocab_size_ != kInvalidSize);
- vocab_pad_ = vocab_pad;
- std::size_t new_size = header_size_ + vocab_size_ + vocab_pad_ + memory_size;
- vocab_string_offset_ = new_size;
- if (!write_mmap_ || write_method_ == Config::WRITE_AFTER) {
- util::HugeMalloc(memory_size, true, memory_search_);
- assert(header_size_ == 0 || write_mmap_);
- vocab_base = reinterpret_cast<uint8_t*>(memory_vocab_.get()) + header_size_;
- util::AdviseHugePages(memory_search_.get(), memory_size);
- return reinterpret_cast<uint8_t*>(memory_search_.get());
- }
-
- assert(write_method_ == Config::WRITE_MMAP);
- // Also known as total size without vocab words.
- // Grow the file to accomodate the search, using zeros.
- // According to man mmap, behavior is undefined when the file is resized
- // underneath a mmap that is not a multiple of the page size. So to be
- // safe, we'll unmap it and map it again.
- mapping_.reset();
- util::ResizeOrThrow(file_.get(), new_size);
- void *ret;
- MapFile(vocab_base, ret);
- util::AdviseHugePages(ret, new_size);
- return ret;
-}
-
-void BinaryFormat::WriteVocabWords(const std::string &buffer, void *&vocab_base, void *&search_base) {
- // Checking Config's include_vocab is the responsibility of the caller.
- assert(header_size_ != kInvalidSize && vocab_size_ != kInvalidSize);
- if (!write_mmap_) {
- // Unchanged base.
- vocab_base = reinterpret_cast<uint8_t*>(memory_vocab_.get());
- search_base = reinterpret_cast<uint8_t*>(memory_search_.get());
- return;
- }
- if (write_method_ == Config::WRITE_MMAP) {
- mapping_.reset();
- }
- util::SeekOrThrow(file_.get(), VocabStringReadingOffset());
- util::WriteOrThrow(file_.get(), &buffer[0], buffer.size());
- if (write_method_ == Config::WRITE_MMAP) {
- MapFile(vocab_base, search_base);
- } else {
- vocab_base = reinterpret_cast<uint8_t*>(memory_vocab_.get()) + header_size_;
- search_base = reinterpret_cast<uint8_t*>(memory_search_.get());
- }
-}
-
-void BinaryFormat::FinishFile(const Config &config, ModelType model_type, unsigned int search_version, const std::vector<uint64_t> &counts) {
- if (!write_mmap_) return;
- switch (write_method_) {
- case Config::WRITE_MMAP:
- util::SyncOrThrow(mapping_.get(), mapping_.size());
- break;
- case Config::WRITE_AFTER:
- util::SeekOrThrow(file_.get(), 0);
- util::WriteOrThrow(file_.get(), memory_vocab_.get(), memory_vocab_.size());
- util::SeekOrThrow(file_.get(), header_size_ + vocab_size_ + vocab_pad_);
- util::WriteOrThrow(file_.get(), memory_search_.get(), memory_search_.size());
- util::FSyncOrThrow(file_.get());
- break;
- }
- // header and vocab share the same mmap.
- Parameters params = Parameters();
- memset(¶ms, 0, sizeof(Parameters));
- params.counts = counts;
- params.fixed.order = counts.size();
- params.fixed.probing_multiplier = config.probing_multiplier;
- params.fixed.model_type = model_type;
- params.fixed.has_vocabulary = config.include_vocab;
- params.fixed.search_version = search_version;
- switch (write_method_) {
- case Config::WRITE_MMAP:
- WriteHeader(mapping_.get(), params);
- util::SyncOrThrow(mapping_.get(), mapping_.size());
- break;
- case Config::WRITE_AFTER:
- {
- std::vector<uint8_t> buffer(TotalHeaderSize(counts.size()));
- WriteHeader(&buffer[0], params);
- util::SeekOrThrow(file_.get(), 0);
- util::WriteOrThrow(file_.get(), &buffer[0], buffer.size());
- }
- break;
- }
-}
-
-void BinaryFormat::MapFile(void *&vocab_base, void *&search_base) {
- mapping_.reset(util::MapOrThrow(vocab_string_offset_, true, util::kFileFlags, false, file_.get()), vocab_string_offset_, util::scoped_memory::MMAP_ALLOCATED);
- vocab_base = reinterpret_cast<uint8_t*>(mapping_.get()) + header_size_;
- search_base = reinterpret_cast<uint8_t*>(mapping_.get()) + header_size_ + vocab_size_ + vocab_pad_;
-}
-
-bool RecognizeBinary(const char *file, ModelType &recognized) {
- util::scoped_fd fd(util::OpenReadOrThrow(file));
- if (!IsBinaryFormat(fd.get())) {
- return false;
- }
- Parameters params;
- ReadHeader(fd.get(), params);
- recognized = params.fixed.model_type;
- return true;
-}
-
-} // namespace ngram
-} // namespace lm
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/binary_format.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/binary_format.hh b/ext/kenlm/lm/binary_format.hh
deleted file mode 100644
index ff99b95..0000000
--- a/ext/kenlm/lm/binary_format.hh
+++ /dev/null
@@ -1,106 +0,0 @@
-#ifndef LM_BINARY_FORMAT_H
-#define LM_BINARY_FORMAT_H
-
-#include "lm/config.hh"
-#include "lm/model_type.hh"
-#include "lm/read_arpa.hh"
-
-#include "util/file_piece.hh"
-#include "util/mmap.hh"
-#include "util/scoped.hh"
-
-#include <cstddef>
-#include <vector>
-
-#include <stdint.h>
-
-namespace lm {
-namespace ngram {
-
-extern const char *kModelNames[6];
-
-/*Inspect a file to determine if it is a binary lm. If not, return false.
- * If so, return true and set recognized to the type. This is the only API in
- * this header designed for use by decoder authors.
- */
-bool RecognizeBinary(const char *file, ModelType &recognized);
-
-struct FixedWidthParameters {
- unsigned char order;
- float probing_multiplier;
- // What type of model is this?
- ModelType model_type;
- // Does the end of the file have the actual strings in the vocabulary?
- bool has_vocabulary;
- unsigned int search_version;
-};
-
-// This is a macro instead of an inline function so constants can be assigned using it.
-#define ALIGN8(a) ((std::ptrdiff_t(((a)-1)/8)+1)*8)
-
-// Parameters stored in the header of a binary file.
-struct Parameters {
- FixedWidthParameters fixed;
- std::vector<uint64_t> counts;
-};
-
-class BinaryFormat {
- public:
- explicit BinaryFormat(const Config &config);
-
- // Reading a binary file:
- // Takes ownership of fd
- void InitializeBinary(int fd, ModelType model_type, unsigned int search_version, Parameters ¶ms);
- // Used to read parts of the file to update the config object before figuring out full size.
- void ReadForConfig(void *to, std::size_t amount, uint64_t offset_excluding_header) const;
- // Actually load the binary file and return a pointer to the beginning of the search area.
- void *LoadBinary(std::size_t size);
-
- uint64_t VocabStringReadingOffset() const {
- assert(vocab_string_offset_ != kInvalidOffset);
- return vocab_string_offset_;
- }
-
- // Writing a binary file or initializing in RAM from ARPA:
- // Size for vocabulary.
- void *SetupJustVocab(std::size_t memory_size, uint8_t order);
- // Warning: can change the vocaulary base pointer.
- void *GrowForSearch(std::size_t memory_size, std::size_t vocab_pad, void *&vocab_base);
- // Warning: can change vocabulary and search base addresses.
- void WriteVocabWords(const std::string &buffer, void *&vocab_base, void *&search_base);
- // Write the header at the beginning of the file.
- void FinishFile(const Config &config, ModelType model_type, unsigned int search_version, const std::vector<uint64_t> &counts);
-
- private:
- void MapFile(void *&vocab_base, void *&search_base);
-
- // Copied from configuration.
- const Config::WriteMethod write_method_;
- const char *write_mmap_;
- util::LoadMethod load_method_;
-
- // File behind memory, if any.
- util::scoped_fd file_;
-
- // If there is a file involved, a single mapping.
- util::scoped_memory mapping_;
-
- // If the data is only in memory, separately allocate each because the trie
- // knows vocab's size before it knows search's size (because SRILM might
- // have pruned).
- util::scoped_memory memory_vocab_, memory_search_;
-
- // Memory ranges. Note that these may not be contiguous and may not all
- // exist.
- std::size_t header_size_, vocab_size_, vocab_pad_;
- // aka end of search.
- uint64_t vocab_string_offset_;
-
- static const uint64_t kInvalidOffset = (uint64_t)-1;
-};
-
-bool IsBinaryFormat(int fd);
-
-} // namespace ngram
-} // namespace lm
-#endif // LM_BINARY_FORMAT_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/blank.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/blank.hh b/ext/kenlm/lm/blank.hh
deleted file mode 100644
index e09054c..0000000
--- a/ext/kenlm/lm/blank.hh
+++ /dev/null
@@ -1,42 +0,0 @@
-#ifndef LM_BLANK_H
-#define LM_BLANK_H
-
-#include <limits>
-#include <stdint.h>
-#include <cmath>
-
-namespace lm {
-namespace ngram {
-
-/* Suppose "foo bar" appears with zero backoff but there is no trigram
- * beginning with these words. Then, when scoring "foo bar", the model could
- * return out_state containing "bar" or even null context if "bar" also has no
- * backoff and is never followed by another word. Then the backoff is set to
- * kNoExtensionBackoff. If the n-gram might be extended, then out_state must
- * contain the full n-gram, in which case kExtensionBackoff is set. In any
- * case, if an n-gram has non-zero backoff, the full state is returned so
- * backoff can be properly charged.
- * These differ only in sign bit because the backoff is in fact zero in either
- * case.
- */
-const float kNoExtensionBackoff = -0.0;
-const float kExtensionBackoff = 0.0;
-const uint64_t kNoExtensionQuant = 0;
-const uint64_t kExtensionQuant = 1;
-
-inline void SetExtension(float &backoff) {
- if (backoff == kNoExtensionBackoff) backoff = kExtensionBackoff;
-}
-
-// This compiles down nicely.
-inline bool HasExtension(const float &backoff) {
- typedef union { float f; uint32_t i; } UnionValue;
- UnionValue compare, interpret;
- compare.f = kNoExtensionBackoff;
- interpret.f = backoff;
- return compare.i != interpret.i;
-}
-
-} // namespace ngram
-} // namespace lm
-#endif // LM_BLANK_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/build_binary_main.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/build_binary_main.cc b/ext/kenlm/lm/build_binary_main.cc
deleted file mode 100644
index 35206e6..0000000
--- a/ext/kenlm/lm/build_binary_main.cc
+++ /dev/null
@@ -1,234 +0,0 @@
-#include "lm/model.hh"
-#include "lm/sizes.hh"
-#include "util/file_piece.hh"
-#include "util/usage.hh"
-
-#include <algorithm>
-#include <cstdlib>
-#include <exception>
-#include <iostream>
-#include <iomanip>
-#include <limits>
-#include <cmath>
-#include <cstdlib>
-
-#ifdef WIN32
-#include "util/getopt.hh"
-#else
-#include <unistd.h>
-#endif
-
-namespace lm {
-namespace ngram {
-namespace {
-
-void Usage(const char *name, const char *default_mem) {
- std::cerr << "Usage: " << name << " [-u log10_unknown_probability] [-s] [-i] [-w mmap|after] [-p probing_multiplier] [-T trie_temporary] [-S trie_building_mem] [-q bits] [-b bits] [-a bits] [type] input.arpa [output.mmap]\n\n"
-"-u sets the log10 probability for <unk> if the ARPA file does not have one.\n"
-" Default is -100. The ARPA file will always take precedence.\n"
-"-s allows models to be built even if they do not have <s> and </s>.\n"
-"-i allows buggy models from IRSTLM by mapping positive log probability to 0.\n"
-"-w mmap|after determines how writing is done.\n"
-" mmap maps the binary file and writes to it. Default for trie.\n"
-" after allocates anonymous memory, builds, and writes. Default for probing.\n"
-"-r \"order1.arpa order2 order3 order4\" adds lower-order rest costs from these\n"
-" model files. order1.arpa must be an ARPA file. All others may be ARPA or\n"
-" the same data structure as being built. All files must have the same\n"
-" vocabulary. For probing, the unigrams must be in the same order.\n\n"
-"type is either probing or trie. Default is probing.\n\n"
-"probing uses a probing hash table. It is the fastest but uses the most memory.\n"
-"-p sets the space multiplier and must be >1.0. The default is 1.5.\n\n"
-"trie is a straightforward trie with bit-level packing. It uses the least\n"
-"memory and is still faster than SRI or IRST. Building the trie format uses an\n"
-"on-disk sort to save memory.\n"
-"-T is the temporary directory prefix. Default is the output file name.\n"
-"-S determines memory use for sorting. Default is " << default_mem << ". This is compatible\n"
-" with GNU sort. The number is followed by a unit: \% for percent of physical\n"
-" memory, b for bytes, K for Kilobytes, M for megabytes, then G,T,P,E,Z,Y. \n"
-" Default unit is K for Kilobytes.\n"
-"-q turns quantization on and sets the number of bits (e.g. -q 8).\n"
-"-b sets backoff quantization bits. Requires -q and defaults to that value.\n"
-"-a compresses pointers using an array of offsets. The parameter is the\n"
-" maximum number of bits encoded by the array. Memory is minimized subject\n"
-" to the maximum, so pick 255 to minimize memory.\n\n"
-"-h print this help message.\n\n"
-"Get a memory estimate by passing an ARPA file without an output file name.\n";
- exit(1);
-}
-
-// I could really use boost::lexical_cast right about now.
-float ParseFloat(const char *from) {
- char *end;
- float ret = strtod(from, &end);
- if (*end) throw util::ParseNumberException(from);
- return ret;
-}
-unsigned long int ParseUInt(const char *from) {
- char *end;
- unsigned long int ret = strtoul(from, &end, 10);
- if (*end) throw util::ParseNumberException(from);
- return ret;
-}
-
-uint8_t ParseBitCount(const char *from) {
- unsigned long val = ParseUInt(from);
- if (val > 25) {
- util::ParseNumberException e(from);
- e << " bit counts are limited to 25.";
- }
- return val;
-}
-
-void ParseFileList(const char *from, std::vector<std::string> &to) {
- to.clear();
- while (true) {
- const char *i;
- for (i = from; *i && *i != ' '; ++i) {}
- to.push_back(std::string(from, i - from));
- if (!*i) break;
- from = i + 1;
- }
-}
-
-void ProbingQuantizationUnsupported() {
- std::cerr << "Quantization is only implemented in the trie data structure." << std::endl;
- exit(1);
-}
-
-} // namespace ngram
-} // namespace lm
-} // namespace
-
-int main(int argc, char *argv[]) {
- using namespace lm::ngram;
-
- const char *default_mem = util::GuessPhysicalMemory() ? "80%" : "1G";
-
- if (argc == 2 && !strcmp(argv[1], "--help"))
- Usage(argv[0], default_mem);
-
- try {
- bool quantize = false, set_backoff_bits = false, bhiksha = false, set_write_method = false, rest = false;
- lm::ngram::Config config;
- config.building_memory = util::ParseSize(default_mem);
- int opt;
- while ((opt = getopt(argc, argv, "q:b:a:u:p:t:T:m:S:w:sir:h")) != -1) {
- switch(opt) {
- case 'q':
- config.prob_bits = ParseBitCount(optarg);
- if (!set_backoff_bits) config.backoff_bits = config.prob_bits;
- quantize = true;
- break;
- case 'b':
- config.backoff_bits = ParseBitCount(optarg);
- set_backoff_bits = true;
- break;
- case 'a':
- config.pointer_bhiksha_bits = ParseBitCount(optarg);
- bhiksha = true;
- break;
- case 'u':
- config.unknown_missing_logprob = ParseFloat(optarg);
- break;
- case 'p':
- config.probing_multiplier = ParseFloat(optarg);
- break;
- case 't': // legacy
- case 'T':
- config.temporary_directory_prefix = optarg;
- util::NormalizeTempPrefix(config.temporary_directory_prefix);
- break;
- case 'm': // legacy
- config.building_memory = ParseUInt(optarg) * 1048576;
- break;
- case 'S':
- config.building_memory = std::min(static_cast<uint64_t>(std::numeric_limits<std::size_t>::max()), util::ParseSize(optarg));
- break;
- case 'w':
- set_write_method = true;
- if (!strcmp(optarg, "mmap")) {
- config.write_method = Config::WRITE_MMAP;
- } else if (!strcmp(optarg, "after")) {
- config.write_method = Config::WRITE_AFTER;
- } else {
- Usage(argv[0], default_mem);
- }
- break;
- case 's':
- config.sentence_marker_missing = lm::SILENT;
- break;
- case 'i':
- config.positive_log_probability = lm::SILENT;
- break;
- case 'r':
- rest = true;
- ParseFileList(optarg, config.rest_lower_files);
- config.rest_function = Config::REST_LOWER;
- break;
- case 'h': // help
- default:
- Usage(argv[0], default_mem);
- }
- }
- if (!quantize && set_backoff_bits) {
- std::cerr << "You specified backoff quantization (-b) but not probability quantization (-q)" << std::endl;
- abort();
- }
- if (optind + 1 == argc) {
- ShowSizes(argv[optind], config);
- return 0;
- }
- const char *model_type;
- const char *from_file;
-
- if (optind + 2 == argc) {
- model_type = "probing";
- from_file = argv[optind];
- config.write_mmap = argv[optind + 1];
- } else if (optind + 3 == argc) {
- model_type = argv[optind];
- from_file = argv[optind + 1];
- config.write_mmap = argv[optind + 2];
- } else {
- Usage(argv[0], default_mem);
- return 1;
- }
- if (!strcmp(model_type, "probing")) {
- if (!set_write_method) config.write_method = Config::WRITE_AFTER;
- if (quantize || set_backoff_bits) ProbingQuantizationUnsupported();
- if (rest) {
- RestProbingModel(from_file, config);
- } else {
- ProbingModel(from_file, config);
- }
- } else if (!strcmp(model_type, "trie")) {
- if (rest) {
- std::cerr << "Rest + trie is not supported yet." << std::endl;
- return 1;
- }
- if (!set_write_method) config.write_method = Config::WRITE_MMAP;
- if (quantize) {
- if (bhiksha) {
- QuantArrayTrieModel(from_file, config);
- } else {
- QuantTrieModel(from_file, config);
- }
- } else {
- if (bhiksha) {
- ArrayTrieModel(from_file, config);
- } else {
- TrieModel(from_file, config);
- }
- }
- } else {
- Usage(argv[0], default_mem);
- }
- }
- catch (const std::exception &e) {
- std::cerr << e.what() << std::endl;
- std::cerr << "ERROR" << std::endl;
- return 1;
- }
- std::cerr << "SUCCESS" << std::endl;
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/builder/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/builder/CMakeLists.txt b/ext/kenlm/lm/builder/CMakeLists.txt
deleted file mode 100644
index cc0d3ed..0000000
--- a/ext/kenlm/lm/builder/CMakeLists.txt
+++ /dev/null
@@ -1,67 +0,0 @@
-cmake_minimum_required(VERSION 2.8.8)
-#
-# The KenLM cmake files make use of add_library(... OBJECTS ...)
-#
-# This syntax allows grouping of source files when compiling
-# (effectively creating "fake" libraries based on source subdirs).
-#
-# This syntax was only added in cmake version 2.8.8
-#
-# see http://www.cmake.org/Wiki/CMake/Tutorials/Object_Library
-
-
-# This CMake file was created by Lane Schwartz <do...@gmail.com>
-
-# Explicitly list the source files for this subdirectory
-#
-# If you add any source files to this subdirectory
-# that should be included in the kenlm library,
-# (this excludes any unit test files)
-# you should add them to the following list:
-#
-# In order to set correct paths to these files
-# in case this variable is referenced by CMake files in the parent directory,
-# we prefix all files with ${CMAKE_CURRENT_SOURCE_DIR}.
-#
-set(KENLM_BUILDER_SOURCE
- ${CMAKE_CURRENT_SOURCE_DIR}/adjust_counts.cc
- ${CMAKE_CURRENT_SOURCE_DIR}/corpus_count.cc
- ${CMAKE_CURRENT_SOURCE_DIR}/initial_probabilities.cc
- ${CMAKE_CURRENT_SOURCE_DIR}/interpolate.cc
- ${CMAKE_CURRENT_SOURCE_DIR}/output.cc
- ${CMAKE_CURRENT_SOURCE_DIR}/pipeline.cc
- )
-
-
-# Group these objects together for later use.
-#
-# Given add_library(foo OBJECT ${my_foo_sources}),
-# refer to these objects as $<TARGET_OBJECTS:foo>
-#
-add_library(kenlm_builder OBJECT ${KENLM_BUILDER_SOURCE})
-
-
-# Compile the executable, linking against the requisite dependent object files
-add_executable(lmplz lmplz_main.cc $<TARGET_OBJECTS:kenlm> $<TARGET_OBJECTS:kenlm_common> $<TARGET_OBJECTS:kenlm_builder> $<TARGET_OBJECTS:kenlm_util>)
-
-# Link the executable against boost
-target_link_libraries(lmplz ${Boost_LIBRARIES} pthread)
-
-# Group executables together
-set_target_properties(lmplz PROPERTIES FOLDER executables)
-
-if(BUILD_TESTING)
-
- # Explicitly list the Boost test files to be compiled
- set(KENLM_BOOST_TESTS_LIST
- adjust_counts_test
- corpus_count_test
- )
-
- AddTests(TESTS ${KENLM_BOOST_TESTS_LIST}
- DEPENDS $<TARGET_OBJECTS:kenlm>
- $<TARGET_OBJECTS:kenlm_common>
- $<TARGET_OBJECTS:kenlm_util>
- $<TARGET_OBJECTS:kenlm_builder>
- LIBRARIES ${Boost_LIBRARIES} pthread)
-endif()
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/builder/Jamfile
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/builder/Jamfile b/ext/kenlm/lm/builder/Jamfile
deleted file mode 100644
index 329a8e0..0000000
--- a/ext/kenlm/lm/builder/Jamfile
+++ /dev/null
@@ -1,13 +0,0 @@
-fakelib builder : [ glob *.cc : *test.cc *main.cc ]
- ../../util//kenutil ../../util/stream//stream ../../util/double-conversion//double-conversion ..//kenlm ../common//common
- : : : <library>/top//boost_thread $(timer-link) ;
-
-exe lmplz : lmplz_main.cc builder /top//boost_program_options ;
-
-exe dump_counts : dump_counts_main.cc builder ;
-
-alias programs : lmplz dump_counts ;
-
-import testing ;
-unit-test corpus_count_test : corpus_count_test.cc builder /top//boost_unit_test_framework ;
-unit-test adjust_counts_test : adjust_counts_test.cc builder /top//boost_unit_test_framework ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/builder/README.md
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/builder/README.md b/ext/kenlm/lm/builder/README.md
deleted file mode 100644
index be0d35e..0000000
--- a/ext/kenlm/lm/builder/README.md
+++ /dev/null
@@ -1,47 +0,0 @@
-Dependencies
-============
-
-Boost >= 1.42.0 is required.
-
-For Ubuntu,
-```bash
-sudo apt-get install libboost1.48-all-dev
-```
-
-Alternatively, you can download, compile, and install it yourself:
-
-```bash
-wget http://sourceforge.net/projects/boost/files/boost/1.52.0/boost_1_52_0.tar.gz/download -O boost_1_52_0.tar.gz
-tar -xvzf boost_1_52_0.tar.gz
-cd boost_1_52_0
-./bootstrap.sh
-./b2
-sudo ./b2 install
-```
-
-Local install options (in a user-space prefix directory) are also possible. See http://www.boost.org/doc/libs/1_52_0/doc/html/bbv2/installation.html.
-
-
-Building
-========
-
-```bash
-bjam
-```
-Your distribution might package bjam and boost-build separately from Boost. Both are required.
-
-Usage
-=====
-
-Run
-```bash
-$ bin/lmplz
-```
-to see command line arguments
-
-Running
-=======
-
-```bash
-bin/lmplz -o 5 <text >text.arpa
-```
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/builder/TODO
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/builder/TODO b/ext/kenlm/lm/builder/TODO
deleted file mode 100644
index cb5aef3..0000000
--- a/ext/kenlm/lm/builder/TODO
+++ /dev/null
@@ -1,5 +0,0 @@
-More tests!
-Sharding.
-Some way to manage all the crazy config options.
-Option to build the binary file directly.
-Interpolation of different orders.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/builder/adjust_counts.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/builder/adjust_counts.cc b/ext/kenlm/lm/builder/adjust_counts.cc
deleted file mode 100644
index b4c5ba8..0000000
--- a/ext/kenlm/lm/builder/adjust_counts.cc
+++ /dev/null
@@ -1,353 +0,0 @@
-#include "lm/builder/adjust_counts.hh"
-#include "lm/common/ngram_stream.hh"
-#include "lm/builder/payload.hh"
-#include "util/stream/timer.hh"
-
-#include <algorithm>
-#include <iostream>
-#include <limits>
-
-namespace lm { namespace builder {
-
-BadDiscountException::BadDiscountException() throw() {}
-BadDiscountException::~BadDiscountException() throw() {}
-
-namespace {
-// Return last word in full that is different.
-const WordIndex* FindDifference(const NGram<BuildingPayload> &full, const NGram<BuildingPayload> &lower_last) {
- const WordIndex *cur_word = full.end() - 1;
- const WordIndex *pre_word = lower_last.end() - 1;
- // Find last difference.
- for (; pre_word >= lower_last.begin() && *pre_word == *cur_word; --cur_word, --pre_word) {}
- return cur_word;
-}
-
-class StatCollector {
- public:
- StatCollector(std::size_t order, std::vector<uint64_t> &counts, std::vector<uint64_t> &counts_pruned, std::vector<Discount> &discounts)
- : orders_(order), full_(orders_.back()), counts_(counts), counts_pruned_(counts_pruned), discounts_(discounts) {
- memset(&orders_[0], 0, sizeof(OrderStat) * order);
- }
-
- ~StatCollector() {}
-
- void CalculateDiscounts(const DiscountConfig &config) {
- counts_.resize(orders_.size());
- counts_pruned_.resize(orders_.size());
- for (std::size_t i = 0; i < orders_.size(); ++i) {
- const OrderStat &s = orders_[i];
- counts_[i] = s.count;
- counts_pruned_[i] = s.count_pruned;
- }
-
- discounts_ = config.overwrite;
- discounts_.resize(orders_.size());
- for (std::size_t i = config.overwrite.size(); i < orders_.size(); ++i) {
- const OrderStat &s = orders_[i];
- try {
- for (unsigned j = 1; j < 4; ++j) {
- // TODO: Specialize error message for j == 3, meaning 3+
- UTIL_THROW_IF(s.n[j] == 0, BadDiscountException, "Could not calculate Kneser-Ney discounts for "
- << (i+1) << "-grams with adjusted count " << (j+1) << " because we didn't observe any "
- << (i+1) << "-grams with adjusted count " << j << "; Is this small or artificial data?\n"
- << "Try deduplicating the input. To override this error for e.g. a class-based model, rerun with --discount_fallback\n");
- }
-
- // See equation (26) in Chen and Goodman.
- discounts_[i].amount[0] = 0.0;
- float y = static_cast<float>(s.n[1]) / static_cast<float>(s.n[1] + 2.0 * s.n[2]);
- for (unsigned j = 1; j < 4; ++j) {
- discounts_[i].amount[j] = static_cast<float>(j) - static_cast<float>(j + 1) * y * static_cast<float>(s.n[j+1]) / static_cast<float>(s.n[j]);
- UTIL_THROW_IF(discounts_[i].amount[j] < 0.0 || discounts_[i].amount[j] > j, BadDiscountException, "ERROR: " << (i+1) << "-gram discount out of range for adjusted count " << j << ": " << discounts_[i].amount[j]);
- }
- } catch (const BadDiscountException &e) {
- switch (config.bad_action) {
- case THROW_UP:
- throw;
- case COMPLAIN:
- std::cerr << "Substituting fallback discounts for order " << i << ": D1=" << config.fallback.amount[1] << " D2=" << config.fallback.amount[2] << " D3+=" << config.fallback.amount[3] << std::endl;
- case SILENT:
- break;
- }
- discounts_[i] = config.fallback;
- }
- }
- }
-
- void Add(std::size_t order_minus_1, uint64_t count, bool pruned = false) {
- OrderStat &stat = orders_[order_minus_1];
- ++stat.count;
- if (!pruned)
- ++stat.count_pruned;
- if (count < 5) ++stat.n[count];
- }
-
- void AddFull(uint64_t count, bool pruned = false) {
- ++full_.count;
- if (!pruned)
- ++full_.count_pruned;
- if (count < 5) ++full_.n[count];
- }
-
- private:
- struct OrderStat {
- // n_1 in equation 26 of Chen and Goodman etc
- uint64_t n[5];
- uint64_t count;
- uint64_t count_pruned;
- };
-
- std::vector<OrderStat> orders_;
- OrderStat &full_;
-
- std::vector<uint64_t> &counts_;
- std::vector<uint64_t> &counts_pruned_;
- std::vector<Discount> &discounts_;
-};
-
-// Reads all entries in order like NGramStream does.
-// But deletes any entries that have <s> in the 1st (not 0th) position on the
-// way out by putting other entries in their place. This disrupts the sort
-// order but we don't care because the data is going to be sorted again.
-class CollapseStream {
- public:
- CollapseStream(const util::stream::ChainPosition &position, uint64_t prune_threshold, const std::vector<bool>& prune_words) :
- current_(NULL, NGram<BuildingPayload>::OrderFromSize(position.GetChain().EntrySize())),
- prune_threshold_(prune_threshold),
- prune_words_(prune_words),
- block_(position) {
- StartBlock();
- }
-
- const NGram<BuildingPayload> &operator*() const { return current_; }
- const NGram<BuildingPayload> *operator->() const { return ¤t_; }
-
- operator bool() const { return block_; }
-
- CollapseStream &operator++() {
- assert(block_);
-
- if (current_.begin()[1] == kBOS && current_.Base() < copy_from_) {
- memcpy(current_.Base(), copy_from_, current_.TotalSize());
- UpdateCopyFrom();
-
- // Mark highest order n-grams for later pruning
- if(current_.Value().count <= prune_threshold_) {
- current_.Value().Mark();
- }
-
- if(!prune_words_.empty()) {
- for(WordIndex* i = current_.begin(); i != current_.end(); i++) {
- if(prune_words_[*i]) {
- current_.Value().Mark();
- break;
- }
- }
- }
-
- }
-
- current_.NextInMemory();
- uint8_t *block_base = static_cast<uint8_t*>(block_->Get());
- if (current_.Base() == block_base + block_->ValidSize()) {
- block_->SetValidSize(copy_from_ + current_.TotalSize() - block_base);
- ++block_;
- StartBlock();
- }
-
- // Mark highest order n-grams for later pruning
- if(current_.Value().count <= prune_threshold_) {
- current_.Value().Mark();
- }
-
- if(!prune_words_.empty()) {
- for(WordIndex* i = current_.begin(); i != current_.end(); i++) {
- if(prune_words_[*i]) {
- current_.Value().Mark();
- break;
- }
- }
- }
-
- return *this;
- }
-
- private:
- void StartBlock() {
- for (; ; ++block_) {
- if (!block_) return;
- if (block_->ValidSize()) break;
- }
- current_.ReBase(block_->Get());
- copy_from_ = static_cast<uint8_t*>(block_->Get()) + block_->ValidSize();
- UpdateCopyFrom();
-
- // Mark highest order n-grams for later pruning
- if(current_.Value().count <= prune_threshold_) {
- current_.Value().Mark();
- }
-
- if(!prune_words_.empty()) {
- for(WordIndex* i = current_.begin(); i != current_.end(); i++) {
- if(prune_words_[*i]) {
- current_.Value().Mark();
- break;
- }
- }
- }
-
- }
-
- // Find last without bos.
- void UpdateCopyFrom() {
- for (copy_from_ -= current_.TotalSize(); copy_from_ >= current_.Base(); copy_from_ -= current_.TotalSize()) {
- if (NGram<BuildingPayload>(copy_from_, current_.Order()).begin()[1] != kBOS) break;
- }
- }
-
- NGram<BuildingPayload> current_;
-
- // Goes backwards in the block
- uint8_t *copy_from_;
- uint64_t prune_threshold_;
- const std::vector<bool>& prune_words_;
- util::stream::Link block_;
-};
-
-} // namespace
-
-void AdjustCounts::Run(const util::stream::ChainPositions &positions) {
- UTIL_TIMER("(%w s) Adjusted counts\n");
-
- const std::size_t order = positions.size();
- StatCollector stats(order, counts_, counts_pruned_, discounts_);
- if (order == 1) {
-
- // Only unigrams. Just collect stats.
- for (NGramStream<BuildingPayload> full(positions[0]); full; ++full) {
-
- // Do not prune <s> </s> <unk>
- if(*full->begin() > 2) {
- if(full->Value().count <= prune_thresholds_[0])
- full->Value().Mark();
-
- if(!prune_words_.empty() && prune_words_[*full->begin()])
- full->Value().Mark();
- }
-
- stats.AddFull(full->Value().UnmarkedCount(), full->Value().IsMarked());
- }
-
- stats.CalculateDiscounts(discount_config_);
- return;
- }
-
- NGramStreams<BuildingPayload> streams;
- streams.Init(positions, positions.size() - 1);
-
- CollapseStream full(positions[positions.size() - 1], prune_thresholds_.back(), prune_words_);
-
- // Initialization: <unk> has count 0 and so does <s>.
- NGramStream<BuildingPayload> *lower_valid = streams.begin();
- const NGramStream<BuildingPayload> *const streams_begin = streams.begin();
- streams[0]->Value().count = 0;
- *streams[0]->begin() = kUNK;
- stats.Add(0, 0);
- (++streams[0])->Value().count = 0;
- *streams[0]->begin() = kBOS;
- // <s> is not in stats yet because it will get put in later.
-
- // This keeps track of actual counts for lower orders. It is not output
- // (only adjusted counts are), but used to determine pruning.
- std::vector<uint64_t> actual_counts(positions.size(), 0);
- // Something of a hack: don't prune <s>.
- actual_counts[0] = std::numeric_limits<uint64_t>::max();
-
- // Iterate over full (the stream of the highest order ngrams)
- for (; full; ++full) {
- const WordIndex *different = FindDifference(*full, **lower_valid);
- std::size_t same = full->end() - 1 - different;
-
- // STEP 1: Output all the n-grams that changed.
- for (; lower_valid >= streams.begin() + same; --lower_valid) {
- uint64_t order_minus_1 = lower_valid - streams_begin;
- if(actual_counts[order_minus_1] <= prune_thresholds_[order_minus_1])
- (*lower_valid)->Value().Mark();
-
- if(!prune_words_.empty()) {
- for(WordIndex* i = (*lower_valid)->begin(); i != (*lower_valid)->end(); i++) {
- if(prune_words_[*i]) {
- (*lower_valid)->Value().Mark();
- break;
- }
- }
- }
-
- stats.Add(order_minus_1, (*lower_valid)->Value().UnmarkedCount(), (*lower_valid)->Value().IsMarked());
- ++*lower_valid;
- }
-
- // STEP 2: Update n-grams that still match.
- // n-grams that match get count from the full entry.
- for (std::size_t i = 0; i < same; ++i) {
- actual_counts[i] += full->Value().UnmarkedCount();
- }
- // Increment the number of unique extensions for the longest match.
- if (same) ++streams[same - 1]->Value().count;
-
- // STEP 3: Initialize new n-grams.
- // This is here because bos is also const WordIndex *, so copy gets
- // consistent argument types.
- const WordIndex *full_end = full->end();
- // Initialize and mark as valid up to bos.
- const WordIndex *bos;
- for (bos = different; (bos > full->begin()) && (*bos != kBOS); --bos) {
- NGramStream<BuildingPayload> &to = *++lower_valid;
- std::copy(bos, full_end, to->begin());
- to->Value().count = 1;
- actual_counts[lower_valid - streams_begin] = full->Value().UnmarkedCount();
- }
- // Now bos indicates where <s> is or is the 0th word of full.
- if (bos != full->begin()) {
- // There is an <s> beyond the 0th word.
- NGramStream<BuildingPayload> &to = *++lower_valid;
- std::copy(bos, full_end, to->begin());
-
- // Anything that begins with <s> has full non adjusted count.
- to->Value().count = full->Value().UnmarkedCount();
- actual_counts[lower_valid - streams_begin] = full->Value().UnmarkedCount();
- } else {
- stats.AddFull(full->Value().UnmarkedCount(), full->Value().IsMarked());
- }
- assert(lower_valid >= &streams[0]);
- }
-
- // The above loop outputs n-grams when it observes changes. This outputs
- // the last n-grams.
- for (NGramStream<BuildingPayload> *s = streams.begin(); s <= lower_valid; ++s) {
- uint64_t lower_count = actual_counts[(*s)->Order() - 1];
- if(lower_count <= prune_thresholds_[(*s)->Order() - 1])
- (*s)->Value().Mark();
-
- if(!prune_words_.empty()) {
- for(WordIndex* i = (*s)->begin(); i != (*s)->end(); i++) {
- if(prune_words_[*i]) {
- (*s)->Value().Mark();
- break;
- }
- }
- }
-
- stats.Add(s - streams.begin(), lower_count, (*s)->Value().IsMarked());
- ++*s;
- }
- // Poison everyone! Except the N-grams which were already poisoned by the input.
- for (NGramStream<BuildingPayload> *s = streams.begin(); s != streams.end(); ++s)
- s->Poison();
-
- stats.CalculateDiscounts(discount_config_);
-
- // NOTE: See special early-return case for unigrams near the top of this function
-}
-
-}} // namespaces
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/builder/adjust_counts.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/builder/adjust_counts.hh b/ext/kenlm/lm/builder/adjust_counts.hh
deleted file mode 100644
index 29319ba..0000000
--- a/ext/kenlm/lm/builder/adjust_counts.hh
+++ /dev/null
@@ -1,72 +0,0 @@
-#ifndef LM_BUILDER_ADJUST_COUNTS_H
-#define LM_BUILDER_ADJUST_COUNTS_H
-
-#include "lm/builder/discount.hh"
-#include "lm/lm_exception.hh"
-#include "util/exception.hh"
-
-#include <vector>
-
-#include <stdint.h>
-
-namespace util { namespace stream { class ChainPositions; } }
-
-namespace lm {
-namespace builder {
-
-class BadDiscountException : public util::Exception {
- public:
- BadDiscountException() throw();
- ~BadDiscountException() throw();
-};
-
-struct DiscountConfig {
- // Overrides discounts for orders [1,discount_override.size()].
- std::vector<Discount> overwrite;
- // If discounting fails for an order, copy them from here.
- Discount fallback;
- // What to do when discounts are out of range or would trigger divison by
- // zero. It it does something other than THROW_UP, use fallback_discount.
- WarningAction bad_action;
-};
-
-/* Compute adjusted counts.
- * Input: unique suffix sorted N-grams (and just the N-grams) with raw counts.
- * Output: [1,N]-grams with adjusted counts.
- * [1,N)-grams are in suffix order
- * N-grams are in undefined order (they're going to be sorted anyway).
- */
-class AdjustCounts {
- public:
- // counts: output
- // counts_pruned: output
- // discounts: mostly output. If the input already has entries, they will be kept.
- // prune_thresholds: input. n-grams with normal (not adjusted) count below this will be pruned.
- AdjustCounts(
- const std::vector<uint64_t> &prune_thresholds,
- std::vector<uint64_t> &counts,
- std::vector<uint64_t> &counts_pruned,
- const std::vector<bool> &prune_words,
- const DiscountConfig &discount_config,
- std::vector<Discount> &discounts)
- : prune_thresholds_(prune_thresholds), counts_(counts), counts_pruned_(counts_pruned),
- prune_words_(prune_words), discount_config_(discount_config), discounts_(discounts)
- {}
-
- void Run(const util::stream::ChainPositions &positions);
-
- private:
- const std::vector<uint64_t> &prune_thresholds_;
- std::vector<uint64_t> &counts_;
- std::vector<uint64_t> &counts_pruned_;
- const std::vector<bool> &prune_words_;
-
- DiscountConfig discount_config_;
- std::vector<Discount> &discounts_;
-};
-
-} // namespace builder
-} // namespace lm
-
-#endif // LM_BUILDER_ADJUST_COUNTS_H
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/builder/adjust_counts_test.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/builder/adjust_counts_test.cc b/ext/kenlm/lm/builder/adjust_counts_test.cc
deleted file mode 100644
index fff551f..0000000
--- a/ext/kenlm/lm/builder/adjust_counts_test.cc
+++ /dev/null
@@ -1,112 +0,0 @@
-#include "lm/builder/adjust_counts.hh"
-
-#include "lm/common/ngram_stream.hh"
-#include "lm/builder/payload.hh"
-#include "util/scoped.hh"
-
-#include <boost/thread/thread.hpp>
-#define BOOST_TEST_MODULE AdjustCounts
-#include <boost/test/unit_test.hpp>
-
-namespace lm { namespace builder { namespace {
-
-class KeepCopy {
- public:
- KeepCopy() : size_(0) {}
-
- void Run(const util::stream::ChainPosition &position) {
- for (util::stream::Link link(position); link; ++link) {
- mem_.call_realloc(size_ + link->ValidSize());
- memcpy(static_cast<uint8_t*>(mem_.get()) + size_, link->Get(), link->ValidSize());
- size_ += link->ValidSize();
- }
- }
-
- uint8_t *Get() { return static_cast<uint8_t*>(mem_.get()); }
- std::size_t Size() const { return size_; }
-
- private:
- util::scoped_malloc mem_;
- std::size_t size_;
-};
-
-struct Gram4 {
- WordIndex ids[4];
- uint64_t count;
-};
-
-class WriteInput {
- public:
- void Run(const util::stream::ChainPosition &position) {
- NGramStream<BuildingPayload> input(position);
- Gram4 grams[] = {
- {{0,0,0,0},10},
- {{0,0,3,0},3},
- // bos
- {{1,1,1,2},5},
- {{0,0,3,2},5},
- };
- for (size_t i = 0; i < sizeof(grams) / sizeof(Gram4); ++i, ++input) {
- memcpy(input->begin(), grams[i].ids, sizeof(WordIndex) * 4);
- input->Value().count = grams[i].count;
- }
- input.Poison();
- }
-};
-
-BOOST_AUTO_TEST_CASE(Simple) {
- KeepCopy outputs[4];
- std::vector<uint64_t> counts;
- std::vector<Discount> discount;
- {
- util::stream::ChainConfig config;
- config.total_memory = 100;
- config.block_count = 1;
- util::stream::Chains chains(4);
- for (unsigned i = 0; i < 4; ++i) {
- config.entry_size = NGram<BuildingPayload>::TotalSize(i + 1);
- chains.push_back(config);
- }
-
- chains[3] >> WriteInput();
- util::stream::ChainPositions for_adjust(chains);
- for (unsigned i = 0; i < 4; ++i) {
- chains[i] >> boost::ref(outputs[i]);
- }
- chains >> util::stream::kRecycle;
- std::vector<uint64_t> counts_pruned(4);
- std::vector<uint64_t> prune_thresholds(4);
- DiscountConfig discount_config;
- discount_config.fallback = Discount();
- discount_config.bad_action = THROW_UP;
- BOOST_CHECK_THROW(AdjustCounts(prune_thresholds, counts, counts_pruned, std::vector<bool>(), discount_config, discount).Run(for_adjust), BadDiscountException);
- }
- BOOST_REQUIRE_EQUAL(4UL, counts.size());
- BOOST_CHECK_EQUAL(4UL, counts[0]);
- // These are no longer set because the discounts are bad.
-/* BOOST_CHECK_EQUAL(4UL, counts[1]);
- BOOST_CHECK_EQUAL(3UL, counts[2]);
- BOOST_CHECK_EQUAL(3UL, counts[3]);*/
- BOOST_REQUIRE_EQUAL(NGram<BuildingPayload>::TotalSize(1) * 4, outputs[0].Size());
- NGram<BuildingPayload> uni(outputs[0].Get(), 1);
- BOOST_CHECK_EQUAL(kUNK, *uni.begin());
- BOOST_CHECK_EQUAL(0ULL, uni.Value().count);
- uni.NextInMemory();
- BOOST_CHECK_EQUAL(kBOS, *uni.begin());
- BOOST_CHECK_EQUAL(0ULL, uni.Value().count);
- uni.NextInMemory();
- BOOST_CHECK_EQUAL(0UL, *uni.begin());
- BOOST_CHECK_EQUAL(2ULL, uni.Value().count);
- uni.NextInMemory();
- BOOST_CHECK_EQUAL(2ULL, uni.Value().count);
- BOOST_CHECK_EQUAL(2UL, *uni.begin());
-
- BOOST_REQUIRE_EQUAL(NGram<BuildingPayload>::TotalSize(2) * 4, outputs[1].Size());
- NGram<BuildingPayload> bi(outputs[1].Get(), 2);
- BOOST_CHECK_EQUAL(0UL, *bi.begin());
- BOOST_CHECK_EQUAL(0UL, *(bi.begin() + 1));
- BOOST_CHECK_EQUAL(1ULL, bi.Value().count);
- bi.NextInMemory();
-}
-
-}}} // namespaces
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/builder/combine_counts.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/builder/combine_counts.hh b/ext/kenlm/lm/builder/combine_counts.hh
deleted file mode 100644
index 2eda517..0000000
--- a/ext/kenlm/lm/builder/combine_counts.hh
+++ /dev/null
@@ -1,31 +0,0 @@
-#ifndef LM_BUILDER_COMBINE_COUNTS_H
-#define LM_BUILDER_COMBINE_COUNTS_H
-
-#include "lm/builder/payload.hh"
-#include "lm/common/ngram.hh"
-#include "lm/common/compare.hh"
-#include "lm/word_index.hh"
-#include "util/stream/sort.hh"
-
-#include <functional>
-#include <string>
-
-namespace lm {
-namespace builder {
-
-// Sum counts for the same n-gram.
-struct CombineCounts {
- bool operator()(void *first_void, const void *second_void, const SuffixOrder &compare) const {
- NGram<BuildingPayload> first(first_void, compare.Order());
- // There isn't a const version of NGram.
- NGram<BuildingPayload> second(const_cast<void*>(second_void), compare.Order());
- if (memcmp(first.begin(), second.begin(), sizeof(WordIndex) * compare.Order())) return false;
- first.Value().count += second.Value().count;
- return true;
- }
-};
-
-} // namespace builder
-} // namespace lm
-
-#endif // LM_BUILDER_COMBINE_COUNTS_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/builder/corpus_count.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/builder/corpus_count.cc b/ext/kenlm/lm/builder/corpus_count.cc
deleted file mode 100644
index 0414c22..0000000
--- a/ext/kenlm/lm/builder/corpus_count.cc
+++ /dev/null
@@ -1,239 +0,0 @@
-#include "lm/builder/corpus_count.hh"
-
-#include "lm/builder/payload.hh"
-#include "lm/common/ngram.hh"
-#include "lm/lm_exception.hh"
-#include "lm/vocab.hh"
-#include "lm/word_index.hh"
-#include "util/file_stream.hh"
-#include "util/file.hh"
-#include "util/file_piece.hh"
-#include "util/murmur_hash.hh"
-#include "util/probing_hash_table.hh"
-#include "util/scoped.hh"
-#include "util/stream/chain.hh"
-#include "util/stream/timer.hh"
-#include "util/tokenize_piece.hh"
-
-#include <functional>
-
-#include <stdint.h>
-
-namespace lm {
-namespace builder {
-namespace {
-
-class DedupeHash : public std::unary_function<const WordIndex *, bool> {
- public:
- explicit DedupeHash(std::size_t order) : size_(order * sizeof(WordIndex)) {}
-
- std::size_t operator()(const WordIndex *start) const {
- return util::MurmurHashNative(start, size_);
- }
-
- private:
- const std::size_t size_;
-};
-
-class DedupeEquals : public std::binary_function<const WordIndex *, const WordIndex *, bool> {
- public:
- explicit DedupeEquals(std::size_t order) : size_(order * sizeof(WordIndex)) {}
-
- bool operator()(const WordIndex *first, const WordIndex *second) const {
- return !memcmp(first, second, size_);
- }
-
- private:
- const std::size_t size_;
-};
-
-struct DedupeEntry {
- typedef WordIndex *Key;
- Key GetKey() const { return key; }
- void SetKey(WordIndex *to) { key = to; }
- Key key;
- static DedupeEntry Construct(WordIndex *at) {
- DedupeEntry ret;
- ret.key = at;
- return ret;
- }
-};
-
-
-// TODO: don't have this here, should be with probing hash table defaults?
-const float kProbingMultiplier = 1.5;
-
-typedef util::ProbingHashTable<DedupeEntry, DedupeHash, DedupeEquals> Dedupe;
-
-class Writer {
- public:
- Writer(std::size_t order, const util::stream::ChainPosition &position, void *dedupe_mem, std::size_t dedupe_mem_size)
- : block_(position), gram_(block_->Get(), order),
- dedupe_invalid_(order, std::numeric_limits<WordIndex>::max()),
- dedupe_(dedupe_mem, dedupe_mem_size, &dedupe_invalid_[0], DedupeHash(order), DedupeEquals(order)),
- buffer_(new WordIndex[order - 1]),
- block_size_(position.GetChain().BlockSize()) {
- dedupe_.Clear();
- assert(Dedupe::Size(position.GetChain().BlockSize() / position.GetChain().EntrySize(), kProbingMultiplier) == dedupe_mem_size);
- if (order == 1) {
- // Add special words. AdjustCounts is responsible if order != 1.
- AddUnigramWord(kUNK);
- AddUnigramWord(kBOS);
- }
- }
-
- ~Writer() {
- block_->SetValidSize(reinterpret_cast<const uint8_t*>(gram_.begin()) - static_cast<const uint8_t*>(block_->Get()));
- (++block_).Poison();
- }
-
- // Write context with a bunch of <s>
- void StartSentence() {
- for (WordIndex *i = gram_.begin(); i != gram_.end() - 1; ++i) {
- *i = kBOS;
- }
- }
-
- void Append(WordIndex word) {
- *(gram_.end() - 1) = word;
- Dedupe::MutableIterator at;
- bool found = dedupe_.FindOrInsert(DedupeEntry::Construct(gram_.begin()), at);
- if (found) {
- // Already present.
- NGram<BuildingPayload> already(at->key, gram_.Order());
- ++(already.Value().count);
- // Shift left by one.
- memmove(gram_.begin(), gram_.begin() + 1, sizeof(WordIndex) * (gram_.Order() - 1));
- return;
- }
- // Complete the write.
- gram_.Value().count = 1;
- // Prepare the next n-gram.
- if (reinterpret_cast<uint8_t*>(gram_.begin()) + gram_.TotalSize() != static_cast<uint8_t*>(block_->Get()) + block_size_) {
- NGram<BuildingPayload> last(gram_);
- gram_.NextInMemory();
- std::copy(last.begin() + 1, last.end(), gram_.begin());
- return;
- }
- // Block end. Need to store the context in a temporary buffer.
- std::copy(gram_.begin() + 1, gram_.end(), buffer_.get());
- dedupe_.Clear();
- block_->SetValidSize(block_size_);
- gram_.ReBase((++block_)->Get());
- std::copy(buffer_.get(), buffer_.get() + gram_.Order() - 1, gram_.begin());
- }
-
- private:
- void AddUnigramWord(WordIndex index) {
- *gram_.begin() = index;
- gram_.Value().count = 0;
- gram_.NextInMemory();
- if (gram_.Base() == static_cast<uint8_t*>(block_->Get()) + block_size_) {
- block_->SetValidSize(block_size_);
- gram_.ReBase((++block_)->Get());
- }
- }
-
- util::stream::Link block_;
-
- NGram<BuildingPayload> gram_;
-
- // This is the memory behind the invalid value in dedupe_.
- std::vector<WordIndex> dedupe_invalid_;
- // Hash table combiner implementation.
- Dedupe dedupe_;
-
- // Small buffer to hold existing ngrams when shifting across a block boundary.
- boost::scoped_array<WordIndex> buffer_;
-
- const std::size_t block_size_;
-};
-
-} // namespace
-
-float CorpusCount::DedupeMultiplier(std::size_t order) {
- return kProbingMultiplier * static_cast<float>(sizeof(DedupeEntry)) / static_cast<float>(NGram<BuildingPayload>::TotalSize(order));
-}
-
-std::size_t CorpusCount::VocabUsage(std::size_t vocab_estimate) {
- return ngram::GrowableVocab<ngram::WriteUniqueWords>::MemUsage(vocab_estimate);
-}
-
-CorpusCount::CorpusCount(util::FilePiece &from, int vocab_write, uint64_t &token_count, WordIndex &type_count, std::vector<bool> &prune_words, const std::string& prune_vocab_filename, std::size_t entries_per_block, WarningAction disallowed_symbol)
- : from_(from), vocab_write_(vocab_write), token_count_(token_count), type_count_(type_count),
- prune_words_(prune_words), prune_vocab_filename_(prune_vocab_filename),
- dedupe_mem_size_(Dedupe::Size(entries_per_block, kProbingMultiplier)),
- dedupe_mem_(util::MallocOrThrow(dedupe_mem_size_)),
- disallowed_symbol_action_(disallowed_symbol) {
-}
-
-namespace {
- void ComplainDisallowed(StringPiece word, WarningAction &action) {
- switch (action) {
- case SILENT:
- return;
- case COMPLAIN:
- std::cerr << "Warning: " << word << " appears in the input. All instances of <s>, </s>, and <unk> will be interpreted as whitespace." << std::endl;
- action = SILENT;
- return;
- case THROW_UP:
- UTIL_THROW(FormatLoadException, "Special word " << word << " is not allowed in the corpus. I plan to support models containing <unk> in the future. Pass --skip_symbols to convert these symbols to whitespace.");
- }
- }
-} // namespace
-
-void CorpusCount::Run(const util::stream::ChainPosition &position) {
- ngram::GrowableVocab<ngram::WriteUniqueWords> vocab(type_count_, vocab_write_);
- token_count_ = 0;
- type_count_ = 0;
- const WordIndex end_sentence = vocab.FindOrInsert("</s>");
- Writer writer(NGram<BuildingPayload>::OrderFromSize(position.GetChain().EntrySize()), position, dedupe_mem_.get(), dedupe_mem_size_);
- uint64_t count = 0;
- bool delimiters[256];
- util::BoolCharacter::Build("\0\t\n\r ", delimiters);
- try {
- while(true) {
- StringPiece line(from_.ReadLine());
- writer.StartSentence();
- for (util::TokenIter<util::BoolCharacter, true> w(line, delimiters); w; ++w) {
- WordIndex word = vocab.FindOrInsert(*w);
- if (word <= 2) {
- ComplainDisallowed(*w, disallowed_symbol_action_);
- continue;
- }
- writer.Append(word);
- ++count;
- }
- writer.Append(end_sentence);
- }
- } catch (const util::EndOfFileException &e) {}
- token_count_ = count;
- type_count_ = vocab.Size();
-
- // Create list of unigrams that are supposed to be pruned
- if (!prune_vocab_filename_.empty()) {
- try {
- util::FilePiece prune_vocab_file(prune_vocab_filename_.c_str());
-
- prune_words_.resize(vocab.Size(), true);
- try {
- while (true) {
- StringPiece word(prune_vocab_file.ReadDelimited(delimiters));
- prune_words_[vocab.Index(word)] = false;
- }
- } catch (const util::EndOfFileException &e) {}
-
- // Never prune <unk>, <s>, </s>
- prune_words_[kUNK] = false;
- prune_words_[kBOS] = false;
- prune_words_[kEOS] = false;
-
- } catch (const util::Exception &e) {
- std::cerr << e.what() << std::endl;
- abort();
- }
- }
-}
-
-} // namespace builder
-} // namespace lm
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/builder/corpus_count.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/builder/corpus_count.hh b/ext/kenlm/lm/builder/corpus_count.hh
deleted file mode 100644
index 165505c..0000000
--- a/ext/kenlm/lm/builder/corpus_count.hh
+++ /dev/null
@@ -1,53 +0,0 @@
-#ifndef LM_BUILDER_CORPUS_COUNT_H
-#define LM_BUILDER_CORPUS_COUNT_H
-
-#include "lm/lm_exception.hh"
-#include "lm/word_index.hh"
-#include "util/scoped.hh"
-
-#include <cstddef>
-#include <string>
-#include <stdint.h>
-#include <vector>
-
-namespace util {
-class FilePiece;
-namespace stream {
-class ChainPosition;
-} // namespace stream
-} // namespace util
-
-namespace lm {
-namespace builder {
-
-class CorpusCount {
- public:
- // Memory usage will be DedupeMultipler(order) * block_size + total_chain_size + unknown vocab_hash_size
- static float DedupeMultiplier(std::size_t order);
-
- // How much memory vocabulary will use based on estimated size of the vocab.
- static std::size_t VocabUsage(std::size_t vocab_estimate);
-
- // token_count: out.
- // type_count aka vocabulary size. Initialize to an estimate. It is set to the exact value.
- CorpusCount(util::FilePiece &from, int vocab_write, uint64_t &token_count, WordIndex &type_count, std::vector<bool> &prune_words, const std::string& prune_vocab_filename, std::size_t entries_per_block, WarningAction disallowed_symbol);
-
- void Run(const util::stream::ChainPosition &position);
-
- private:
- util::FilePiece &from_;
- int vocab_write_;
- uint64_t &token_count_;
- WordIndex &type_count_;
- std::vector<bool>& prune_words_;
- const std::string& prune_vocab_filename_;
-
- std::size_t dedupe_mem_size_;
- util::scoped_malloc dedupe_mem_;
-
- WarningAction disallowed_symbol_action_;
-};
-
-} // namespace builder
-} // namespace lm
-#endif // LM_BUILDER_CORPUS_COUNT_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/builder/corpus_count_test.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/builder/corpus_count_test.cc b/ext/kenlm/lm/builder/corpus_count_test.cc
deleted file mode 100644
index 88bcf96..0000000
--- a/ext/kenlm/lm/builder/corpus_count_test.cc
+++ /dev/null
@@ -1,79 +0,0 @@
-#include "lm/builder/corpus_count.hh"
-
-#include "lm/builder/payload.hh"
-#include "lm/common/ngram_stream.hh"
-#include "lm/common/ngram.hh"
-
-#include "util/file.hh"
-#include "util/file_piece.hh"
-#include "util/tokenize_piece.hh"
-#include "util/stream/chain.hh"
-#include "util/stream/stream.hh"
-
-#define BOOST_TEST_MODULE CorpusCountTest
-#include <boost/test/unit_test.hpp>
-
-namespace lm { namespace builder { namespace {
-
-#define Check(str, cnt) { \
- BOOST_REQUIRE(stream); \
- w = stream->begin(); \
- for (util::TokenIter<util::AnyCharacter, true> t(str, " "); t; ++t, ++w) { \
- BOOST_CHECK_EQUAL(*t, v[*w]); \
- } \
- BOOST_CHECK_EQUAL((uint64_t)cnt, stream->Value().count); \
- ++stream; \
-}
-
-BOOST_AUTO_TEST_CASE(Short) {
- util::scoped_fd input_file(util::MakeTemp("corpus_count_test_temp"));
- const char input[] = "looking on a little more loin\non a little more loin\non foo little more loin\nbar\n\n";
- // Blocks of 10 are
- // looking on a little more loin </s> on a little[duplicate] more[duplicate] loin[duplicate] </s>[duplicate] on[duplicate] foo
- // little more loin </s> bar </s> </s>
-
- util::WriteOrThrow(input_file.get(), input, sizeof(input) - 1);
- util::FilePiece input_piece(input_file.release(), "temp file");
-
- util::stream::ChainConfig config;
- config.entry_size = NGram<BuildingPayload>::TotalSize(3);
- config.total_memory = config.entry_size * 20;
- config.block_count = 2;
-
- util::scoped_fd vocab(util::MakeTemp("corpus_count_test_vocab"));
-
- util::stream::Chain chain(config);
- uint64_t token_count;
- WordIndex type_count = 10;
- std::vector<bool> prune_words;
- CorpusCount counter(input_piece, vocab.get(), token_count, type_count, prune_words, "", chain.BlockSize() / chain.EntrySize(), SILENT);
- chain >> boost::ref(counter);
- NGramStream<BuildingPayload> stream(chain.Add());
- chain >> util::stream::kRecycle;
-
- const char *v[] = {"<unk>", "<s>", "</s>", "looking", "on", "a", "little", "more", "loin", "foo", "bar"};
-
- WordIndex *w;
-
- Check("<s> <s> looking", 1);
- Check("<s> looking on", 1);
- Check("looking on a", 1);
- Check("on a little", 2);
- Check("a little more", 2);
- Check("little more loin", 2);
- Check("more loin </s>", 2);
- Check("<s> <s> on", 2);
- Check("<s> on a", 1);
- Check("<s> on foo", 1);
- Check("on foo little", 1);
- Check("foo little more", 1);
- Check("little more loin", 1);
- Check("more loin </s>", 1);
- Check("<s> <s> bar", 1);
- Check("<s> bar </s>", 1);
- Check("<s> <s> </s>", 1);
- BOOST_CHECK(!stream);
- BOOST_CHECK_EQUAL(sizeof(v) / sizeof(const char*), type_count);
-}
-
-}}} // namespaces
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/builder/debug_print.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/builder/debug_print.hh b/ext/kenlm/lm/builder/debug_print.hh
deleted file mode 100644
index 4b9f306..0000000
--- a/ext/kenlm/lm/builder/debug_print.hh
+++ /dev/null
@@ -1,70 +0,0 @@
-#ifndef LM_BUILDER_DEBUG_PRINT_H
-#define LM_BUILDER_DEBUG_PRINT_H
-
-#include "lm/builder/payload.hh"
-#include "lm/common/print.hh"
-#include "lm/common/ngram_stream.hh"
-#include "util/file_stream.hh"
-#include "util/file.hh"
-
-#include <boost/lexical_cast.hpp>
-
-namespace lm { namespace builder {
-// Not defined, only specialized.
-template <class T> void PrintPayload(util::FileStream &to, const BuildingPayload &payload);
-template <> inline void PrintPayload<uint64_t>(util::FileStream &to, const BuildingPayload &payload) {
- to << payload.count;
-}
-template <> inline void PrintPayload<Uninterpolated>(util::FileStream &to, const BuildingPayload &payload) {
- to << log10(payload.uninterp.prob) << ' ' << log10(payload.uninterp.gamma);
-}
-template <> inline void PrintPayload<ProbBackoff>(util::FileStream &to, const BuildingPayload &payload) {
- to << payload.complete.prob << ' ' << payload.complete.backoff;
-}
-
-// template parameter is the type stored.
-template <class V> class Print {
- public:
- static void DumpSeparateFiles(const VocabReconstitute &vocab, const std::string &file_base, util::stream::Chains &chains) {
- for (unsigned int i = 0; i < chains.size(); ++i) {
- std::string file(file_base + boost::lexical_cast<std::string>(i));
- chains[i] >> Print(vocab, util::CreateOrThrow(file.c_str()));
- }
- }
-
- explicit Print(const VocabReconstitute &vocab, int fd) : vocab_(vocab), to_(fd) {}
-
- void Run(const util::stream::ChainPositions &chains) {
- util::scoped_fd fd(to_);
- util::FileStream out(to_);
- NGramStreams<BuildingPayload> streams(chains);
- for (NGramStream<BuildingPayload> *s = streams.begin(); s != streams.end(); ++s) {
- DumpStream(*s, out);
- }
- }
-
- void Run(const util::stream::ChainPosition &position) {
- util::scoped_fd fd(to_);
- util::FileStream out(to_);
- NGramStream<BuildingPayload> stream(position);
- DumpStream(stream, out);
- }
-
- private:
- void DumpStream(NGramStream<BuildingPayload> &stream, util::FileStream &to) {
- for (; stream; ++stream) {
- PrintPayload<V>(to, stream->Value());
- for (const WordIndex *w = stream->begin(); w != stream->end(); ++w) {
- to << ' ' << vocab_.Lookup(*w) << '=' << *w;
- }
- to << '\n';
- }
- }
-
- const VocabReconstitute &vocab_;
- int to_;
-};
-
-}} // namespaces
-
-#endif // LM_BUILDER_DEBUG_PRINT_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/builder/discount.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/builder/discount.hh b/ext/kenlm/lm/builder/discount.hh
deleted file mode 100644
index e2f4084..0000000
--- a/ext/kenlm/lm/builder/discount.hh
+++ /dev/null
@@ -1,26 +0,0 @@
-#ifndef LM_BUILDER_DISCOUNT_H
-#define LM_BUILDER_DISCOUNT_H
-
-#include <algorithm>
-
-#include <stdint.h>
-
-namespace lm {
-namespace builder {
-
-struct Discount {
- float amount[4];
-
- float Get(uint64_t count) const {
- return amount[std::min<uint64_t>(count, 3)];
- }
-
- float Apply(uint64_t count) const {
- return static_cast<float>(count) - Get(count);
- }
-};
-
-} // namespace builder
-} // namespace lm
-
-#endif // LM_BUILDER_DISCOUNT_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/builder/dump_counts_main.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/builder/dump_counts_main.cc b/ext/kenlm/lm/builder/dump_counts_main.cc
deleted file mode 100644
index 26078d0..0000000
--- a/ext/kenlm/lm/builder/dump_counts_main.cc
+++ /dev/null
@@ -1,36 +0,0 @@
-#include "lm/common/print.hh"
-#include "lm/word_index.hh"
-#include "util/file.hh"
-#include "util/read_compressed.hh"
-
-#include <boost/lexical_cast.hpp>
-
-#include <iostream>
-#include <vector>
-
-int main(int argc, char *argv[]) {
- if (argc != 4) {
- std::cerr << "Usage: " << argv[0] << " counts vocabulary order\n"
- "The counts file contains records with 4-byte vocabulary ids followed by 8-byte\n"
- "counts. Each record has order many vocabulary ids.\n"
- "The vocabulary file contains the words delimited by NULL in order of id.\n"
- "The vocabulary file may not be compressed because it is mmapped but the counts\n"
- "file can be compressed.\n";
- return 1;
- }
- util::ReadCompressed counts(util::OpenReadOrThrow(argv[1]));
- util::scoped_fd vocab_file(util::OpenReadOrThrow(argv[2]));
- lm::VocabReconstitute vocab(vocab_file.get());
- unsigned int order = boost::lexical_cast<unsigned int>(argv[3]);
- std::vector<char> record(sizeof(uint32_t) * order + sizeof(uint64_t));
- while (std::size_t got = counts.ReadOrEOF(&*record.begin(), record.size())) {
- UTIL_THROW_IF(got != record.size(), util::Exception, "Read " << got << " bytes at the end of file, which is not a complete record of length " << record.size());
- const lm::WordIndex *words = reinterpret_cast<const lm::WordIndex*>(&*record.begin());
- for (const lm::WordIndex *i = words; i != words + order; ++i) {
- UTIL_THROW_IF(*i >= vocab.Size(), util::Exception, "Vocab ID " << *i << " is larger than the vocab file's maximum of " << vocab.Size() << ". Are you sure you have the right order and vocab file for these counts?");
- std::cout << vocab.Lookup(*i) << ' ';
- }
- // TODO don't use std::cout because it is slow. Add fast uint64_t printing support to FileStream.
- std::cout << *reinterpret_cast<const uint64_t*>(words + order) << '\n';
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/builder/hash_gamma.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/builder/hash_gamma.hh b/ext/kenlm/lm/builder/hash_gamma.hh
deleted file mode 100644
index 4bef47e..0000000
--- a/ext/kenlm/lm/builder/hash_gamma.hh
+++ /dev/null
@@ -1,19 +0,0 @@
-#ifndef LM_BUILDER_HASH_GAMMA__
-#define LM_BUILDER_HASH_GAMMA__
-
-#include <stdint.h>
-
-namespace lm { namespace builder {
-
-#pragma pack(push)
-#pragma pack(4)
-
-struct HashGamma {
- uint64_t hash_value;
- float gamma;
-};
-
-#pragma pack(pop)
-
-}} // namespaces
-#endif // LM_BUILDER_HASH_GAMMA__
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/builder/header_info.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/builder/header_info.hh b/ext/kenlm/lm/builder/header_info.hh
deleted file mode 100644
index d01d049..0000000
--- a/ext/kenlm/lm/builder/header_info.hh
+++ /dev/null
@@ -1,28 +0,0 @@
-#ifndef LM_BUILDER_HEADER_INFO_H
-#define LM_BUILDER_HEADER_INFO_H
-
-#include <string>
-#include <vector>
-#include <stdint.h>
-
-namespace lm { namespace builder {
-
-// Some configuration info that is used to add
-// comments to the beginning of an ARPA file
-struct HeaderInfo {
- std::string input_file;
- uint64_t token_count;
- std::vector<uint64_t> counts_pruned;
-
- HeaderInfo() {}
-
- HeaderInfo(const std::string& input_file_in, uint64_t token_count_in, const std::vector<uint64_t> &counts_pruned_in)
- : input_file(input_file_in), token_count(token_count_in), counts_pruned(counts_pruned_in) {}
-
- // TODO: Add smoothing type
- // TODO: More info if multiple models were interpolated
-};
-
-}} // namespaces
-
-#endif
[02/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/double-conversion/strtod.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/double-conversion/strtod.cc b/ext/kenlm/util/double-conversion/strtod.cc
deleted file mode 100644
index 55b4daa..0000000
--- a/ext/kenlm/util/double-conversion/strtod.cc
+++ /dev/null
@@ -1,558 +0,0 @@
-// Copyright 2010 the V8 project authors. All rights reserved.
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following
-// disclaimer in the documentation and/or other materials provided
-// with the distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived
-// from this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#include <cstdarg>
-#include <climits>
-
-#include "strtod.h"
-#include "bignum.h"
-#include "cached-powers.h"
-#include "ieee.h"
-
-namespace double_conversion {
-
-// 2^53 = 9007199254740992.
-// Any integer with at most 15 decimal digits will hence fit into a double
-// (which has a 53bit significand) without loss of precision.
-static const int kMaxExactDoubleIntegerDecimalDigits = 15;
-// 2^64 = 18446744073709551616 > 10^19
-static const int kMaxUint64DecimalDigits = 19;
-
-// Max double: 1.7976931348623157 x 10^308
-// Min non-zero double: 4.9406564584124654 x 10^-324
-// Any x >= 10^309 is interpreted as +infinity.
-// Any x <= 10^-324 is interpreted as 0.
-// Note that 2.5e-324 (despite being smaller than the min double) will be read
-// as non-zero (equal to the min non-zero double).
-static const int kMaxDecimalPower = 309;
-static const int kMinDecimalPower = -324;
-
-// 2^64 = 18446744073709551616
-static const uint64_t kMaxUint64 = UINT64_2PART_C(0xFFFFFFFF, FFFFFFFF);
-
-
-static const double exact_powers_of_ten[] = {
- 1.0, // 10^0
- 10.0,
- 100.0,
- 1000.0,
- 10000.0,
- 100000.0,
- 1000000.0,
- 10000000.0,
- 100000000.0,
- 1000000000.0,
- 10000000000.0, // 10^10
- 100000000000.0,
- 1000000000000.0,
- 10000000000000.0,
- 100000000000000.0,
- 1000000000000000.0,
- 10000000000000000.0,
- 100000000000000000.0,
- 1000000000000000000.0,
- 10000000000000000000.0,
- 100000000000000000000.0, // 10^20
- 1000000000000000000000.0,
- // 10^22 = 0x21e19e0c9bab2400000 = 0x878678326eac9 * 2^22
- 10000000000000000000000.0
-};
-static const int kExactPowersOfTenSize = ARRAY_SIZE(exact_powers_of_ten);
-
-// Maximum number of significant digits in the decimal representation.
-// In fact the value is 772 (see conversions.cc), but to give us some margin
-// we round up to 780.
-static const int kMaxSignificantDecimalDigits = 780;
-
-static Vector<const char> TrimLeadingZeros(Vector<const char> buffer) {
- for (int i = 0; i < buffer.length(); i++) {
- if (buffer[i] != '0') {
- return buffer.SubVector(i, buffer.length());
- }
- }
- return Vector<const char>(buffer.start(), 0);
-}
-
-
-static Vector<const char> TrimTrailingZeros(Vector<const char> buffer) {
- for (int i = buffer.length() - 1; i >= 0; --i) {
- if (buffer[i] != '0') {
- return buffer.SubVector(0, i + 1);
- }
- }
- return Vector<const char>(buffer.start(), 0);
-}
-
-
-static void CutToMaxSignificantDigits(Vector<const char> buffer,
- int exponent,
- char* significant_buffer,
- int* significant_exponent) {
- for (int i = 0; i < kMaxSignificantDecimalDigits - 1; ++i) {
- significant_buffer[i] = buffer[i];
- }
- // The input buffer has been trimmed. Therefore the last digit must be
- // different from '0'.
- ASSERT(buffer[buffer.length() - 1] != '0');
- // Set the last digit to be non-zero. This is sufficient to guarantee
- // correct rounding.
- significant_buffer[kMaxSignificantDecimalDigits - 1] = '1';
- *significant_exponent =
- exponent + (buffer.length() - kMaxSignificantDecimalDigits);
-}
-
-
-// Trims the buffer and cuts it to at most kMaxSignificantDecimalDigits.
-// If possible the input-buffer is reused, but if the buffer needs to be
-// modified (due to cutting), then the input needs to be copied into the
-// buffer_copy_space.
-static void TrimAndCut(Vector<const char> buffer, int exponent,
- char* buffer_copy_space, int space_size,
- Vector<const char>* trimmed, int* updated_exponent) {
- Vector<const char> left_trimmed = TrimLeadingZeros(buffer);
- Vector<const char> right_trimmed = TrimTrailingZeros(left_trimmed);
- exponent += left_trimmed.length() - right_trimmed.length();
- if (right_trimmed.length() > kMaxSignificantDecimalDigits) {
- ASSERT(space_size >= kMaxSignificantDecimalDigits);
- CutToMaxSignificantDigits(right_trimmed, exponent,
- buffer_copy_space, updated_exponent);
- *trimmed = Vector<const char>(buffer_copy_space,
- kMaxSignificantDecimalDigits);
- } else {
- *trimmed = right_trimmed;
- *updated_exponent = exponent;
- }
-}
-
-
-// Reads digits from the buffer and converts them to a uint64.
-// Reads in as many digits as fit into a uint64.
-// When the string starts with "1844674407370955161" no further digit is read.
-// Since 2^64 = 18446744073709551616 it would still be possible read another
-// digit if it was less or equal than 6, but this would complicate the code.
-static uint64_t ReadUint64(Vector<const char> buffer,
- int* number_of_read_digits) {
- uint64_t result = 0;
- int i = 0;
- while (i < buffer.length() && result <= (kMaxUint64 / 10 - 1)) {
- int digit = buffer[i++] - '0';
- ASSERT(0 <= digit && digit <= 9);
- result = 10 * result + digit;
- }
- *number_of_read_digits = i;
- return result;
-}
-
-
-// Reads a DiyFp from the buffer.
-// The returned DiyFp is not necessarily normalized.
-// If remaining_decimals is zero then the returned DiyFp is accurate.
-// Otherwise it has been rounded and has error of at most 1/2 ulp.
-static void ReadDiyFp(Vector<const char> buffer,
- DiyFp* result,
- int* remaining_decimals) {
- int read_digits;
- uint64_t significand = ReadUint64(buffer, &read_digits);
- if (buffer.length() == read_digits) {
- *result = DiyFp(significand, 0);
- *remaining_decimals = 0;
- } else {
- // Round the significand.
- if (buffer[read_digits] >= '5') {
- significand++;
- }
- // Compute the binary exponent.
- int exponent = 0;
- *result = DiyFp(significand, exponent);
- *remaining_decimals = buffer.length() - read_digits;
- }
-}
-
-
-static bool DoubleStrtod(Vector<const char> trimmed,
- int exponent,
- double* result) {
-#if !defined(DOUBLE_CONVERSION_CORRECT_DOUBLE_OPERATIONS)
- // On x86 the floating-point stack can be 64 or 80 bits wide. If it is
- // 80 bits wide (as is the case on Linux) then double-rounding occurs and the
- // result is not accurate.
- // We know that Windows32 uses 64 bits and is therefore accurate.
- // Note that the ARM simulator is compiled for 32bits. It therefore exhibits
- // the same problem.
- return false;
-#endif
- if (trimmed.length() <= kMaxExactDoubleIntegerDecimalDigits) {
- int read_digits;
- // The trimmed input fits into a double.
- // If the 10^exponent (resp. 10^-exponent) fits into a double too then we
- // can compute the result-double simply by multiplying (resp. dividing) the
- // two numbers.
- // This is possible because IEEE guarantees that floating-point operations
- // return the best possible approximation.
- if (exponent < 0 && -exponent < kExactPowersOfTenSize) {
- // 10^-exponent fits into a double.
- *result = static_cast<double>(ReadUint64(trimmed, &read_digits));
- ASSERT(read_digits == trimmed.length());
- *result /= exact_powers_of_ten[-exponent];
- return true;
- }
- if (0 <= exponent && exponent < kExactPowersOfTenSize) {
- // 10^exponent fits into a double.
- *result = static_cast<double>(ReadUint64(trimmed, &read_digits));
- ASSERT(read_digits == trimmed.length());
- *result *= exact_powers_of_ten[exponent];
- return true;
- }
- int remaining_digits =
- kMaxExactDoubleIntegerDecimalDigits - trimmed.length();
- if ((0 <= exponent) &&
- (exponent - remaining_digits < kExactPowersOfTenSize)) {
- // The trimmed string was short and we can multiply it with
- // 10^remaining_digits. As a result the remaining exponent now fits
- // into a double too.
- *result = static_cast<double>(ReadUint64(trimmed, &read_digits));
- ASSERT(read_digits == trimmed.length());
- *result *= exact_powers_of_ten[remaining_digits];
- *result *= exact_powers_of_ten[exponent - remaining_digits];
- return true;
- }
- }
- return false;
-}
-
-
-// Returns 10^exponent as an exact DiyFp.
-// The given exponent must be in the range [1; kDecimalExponentDistance[.
-static DiyFp AdjustmentPowerOfTen(int exponent) {
- ASSERT(0 < exponent);
- ASSERT(exponent < PowersOfTenCache::kDecimalExponentDistance);
- // Simply hardcode the remaining powers for the given decimal exponent
- // distance.
- ASSERT(PowersOfTenCache::kDecimalExponentDistance == 8);
- switch (exponent) {
- case 1: return DiyFp(UINT64_2PART_C(0xa0000000, 00000000), -60);
- case 2: return DiyFp(UINT64_2PART_C(0xc8000000, 00000000), -57);
- case 3: return DiyFp(UINT64_2PART_C(0xfa000000, 00000000), -54);
- case 4: return DiyFp(UINT64_2PART_C(0x9c400000, 00000000), -50);
- case 5: return DiyFp(UINT64_2PART_C(0xc3500000, 00000000), -47);
- case 6: return DiyFp(UINT64_2PART_C(0xf4240000, 00000000), -44);
- case 7: return DiyFp(UINT64_2PART_C(0x98968000, 00000000), -40);
- default:
- UNREACHABLE();
- return DiyFp(0, 0);
- }
-}
-
-
-// If the function returns true then the result is the correct double.
-// Otherwise it is either the correct double or the double that is just below
-// the correct double.
-static bool DiyFpStrtod(Vector<const char> buffer,
- int exponent,
- double* result) {
- DiyFp input;
- int remaining_decimals;
- ReadDiyFp(buffer, &input, &remaining_decimals);
- // Since we may have dropped some digits the input is not accurate.
- // If remaining_decimals is different than 0 than the error is at most
- // .5 ulp (unit in the last place).
- // We don't want to deal with fractions and therefore keep a common
- // denominator.
- const int kDenominatorLog = 3;
- const int kDenominator = 1 << kDenominatorLog;
- // Move the remaining decimals into the exponent.
- exponent += remaining_decimals;
- int error = (remaining_decimals == 0 ? 0 : kDenominator / 2);
-
- int old_e = input.e();
- input.Normalize();
- error <<= old_e - input.e();
-
- ASSERT(exponent <= PowersOfTenCache::kMaxDecimalExponent);
- if (exponent < PowersOfTenCache::kMinDecimalExponent) {
- *result = 0.0;
- return true;
- }
- DiyFp cached_power;
- int cached_decimal_exponent;
- PowersOfTenCache::GetCachedPowerForDecimalExponent(exponent,
- &cached_power,
- &cached_decimal_exponent);
-
- if (cached_decimal_exponent != exponent) {
- int adjustment_exponent = exponent - cached_decimal_exponent;
- DiyFp adjustment_power = AdjustmentPowerOfTen(adjustment_exponent);
- input.Multiply(adjustment_power);
- if (kMaxUint64DecimalDigits - buffer.length() >= adjustment_exponent) {
- // The product of input with the adjustment power fits into a 64 bit
- // integer.
- ASSERT(DiyFp::kSignificandSize == 64);
- } else {
- // The adjustment power is exact. There is hence only an error of 0.5.
- error += kDenominator / 2;
- }
- }
-
- input.Multiply(cached_power);
- // The error introduced by a multiplication of a*b equals
- // error_a + error_b + error_a*error_b/2^64 + 0.5
- // Substituting a with 'input' and b with 'cached_power' we have
- // error_b = 0.5 (all cached powers have an error of less than 0.5 ulp),
- // error_ab = 0 or 1 / kDenominator > error_a*error_b/ 2^64
- int error_b = kDenominator / 2;
- int error_ab = (error == 0 ? 0 : 1); // We round up to 1.
- int fixed_error = kDenominator / 2;
- error += error_b + error_ab + fixed_error;
-
- old_e = input.e();
- input.Normalize();
- error <<= old_e - input.e();
-
- // See if the double's significand changes if we add/subtract the error.
- int order_of_magnitude = DiyFp::kSignificandSize + input.e();
- int effective_significand_size =
- Double::SignificandSizeForOrderOfMagnitude(order_of_magnitude);
- int precision_digits_count =
- DiyFp::kSignificandSize - effective_significand_size;
- if (precision_digits_count + kDenominatorLog >= DiyFp::kSignificandSize) {
- // This can only happen for very small denormals. In this case the
- // half-way multiplied by the denominator exceeds the range of an uint64.
- // Simply shift everything to the right.
- int shift_amount = (precision_digits_count + kDenominatorLog) -
- DiyFp::kSignificandSize + 1;
- input.set_f(input.f() >> shift_amount);
- input.set_e(input.e() + shift_amount);
- // We add 1 for the lost precision of error, and kDenominator for
- // the lost precision of input.f().
- error = (error >> shift_amount) + 1 + kDenominator;
- precision_digits_count -= shift_amount;
- }
- // We use uint64_ts now. This only works if the DiyFp uses uint64_ts too.
- ASSERT(DiyFp::kSignificandSize == 64);
- ASSERT(precision_digits_count < 64);
- uint64_t one64 = 1;
- uint64_t precision_bits_mask = (one64 << precision_digits_count) - 1;
- uint64_t precision_bits = input.f() & precision_bits_mask;
- uint64_t half_way = one64 << (precision_digits_count - 1);
- precision_bits *= kDenominator;
- half_way *= kDenominator;
- DiyFp rounded_input(input.f() >> precision_digits_count,
- input.e() + precision_digits_count);
- if (precision_bits >= half_way + error) {
- rounded_input.set_f(rounded_input.f() + 1);
- }
- // If the last_bits are too close to the half-way case than we are too
- // inaccurate and round down. In this case we return false so that we can
- // fall back to a more precise algorithm.
-
- *result = Double(rounded_input).value();
- if (half_way - error < precision_bits && precision_bits < half_way + error) {
- // Too imprecise. The caller will have to fall back to a slower version.
- // However the returned number is guaranteed to be either the correct
- // double, or the next-lower double.
- return false;
- } else {
- return true;
- }
-}
-
-
-// Returns
-// - -1 if buffer*10^exponent < diy_fp.
-// - 0 if buffer*10^exponent == diy_fp.
-// - +1 if buffer*10^exponent > diy_fp.
-// Preconditions:
-// buffer.length() + exponent <= kMaxDecimalPower + 1
-// buffer.length() + exponent > kMinDecimalPower
-// buffer.length() <= kMaxDecimalSignificantDigits
-static int CompareBufferWithDiyFp(Vector<const char> buffer,
- int exponent,
- DiyFp diy_fp) {
- ASSERT(buffer.length() + exponent <= kMaxDecimalPower + 1);
- ASSERT(buffer.length() + exponent > kMinDecimalPower);
- ASSERT(buffer.length() <= kMaxSignificantDecimalDigits);
- // Make sure that the Bignum will be able to hold all our numbers.
- // Our Bignum implementation has a separate field for exponents. Shifts will
- // consume at most one bigit (< 64 bits).
- // ln(10) == 3.3219...
- ASSERT(((kMaxDecimalPower + 1) * 333 / 100) < Bignum::kMaxSignificantBits);
- Bignum buffer_bignum;
- Bignum diy_fp_bignum;
- buffer_bignum.AssignDecimalString(buffer);
- diy_fp_bignum.AssignUInt64(diy_fp.f());
- if (exponent >= 0) {
- buffer_bignum.MultiplyByPowerOfTen(exponent);
- } else {
- diy_fp_bignum.MultiplyByPowerOfTen(-exponent);
- }
- if (diy_fp.e() > 0) {
- diy_fp_bignum.ShiftLeft(diy_fp.e());
- } else {
- buffer_bignum.ShiftLeft(-diy_fp.e());
- }
- return Bignum::Compare(buffer_bignum, diy_fp_bignum);
-}
-
-
-// Returns true if the guess is the correct double.
-// Returns false, when guess is either correct or the next-lower double.
-static bool ComputeGuess(Vector<const char> trimmed, int exponent,
- double* guess) {
- if (trimmed.length() == 0) {
- *guess = 0.0;
- return true;
- }
- if (exponent + trimmed.length() - 1 >= kMaxDecimalPower) {
- *guess = Double::Infinity();
- return true;
- }
- if (exponent + trimmed.length() <= kMinDecimalPower) {
- *guess = 0.0;
- return true;
- }
-
- if (DoubleStrtod(trimmed, exponent, guess) ||
- DiyFpStrtod(trimmed, exponent, guess)) {
- return true;
- }
- if (*guess == Double::Infinity()) {
- return true;
- }
- return false;
-}
-
-double Strtod(Vector<const char> buffer, int exponent) {
- char copy_buffer[kMaxSignificantDecimalDigits];
- Vector<const char> trimmed;
- int updated_exponent;
- TrimAndCut(buffer, exponent, copy_buffer, kMaxSignificantDecimalDigits,
- &trimmed, &updated_exponent);
- exponent = updated_exponent;
-
- double guess;
- bool is_correct = ComputeGuess(trimmed, exponent, &guess);
- if (is_correct) return guess;
-
- DiyFp upper_boundary = Double(guess).UpperBoundary();
- int comparison = CompareBufferWithDiyFp(trimmed, exponent, upper_boundary);
- if (comparison < 0) {
- return guess;
- } else if (comparison > 0) {
- return Double(guess).NextDouble();
- } else if ((Double(guess).Significand() & 1) == 0) {
- // Round towards even.
- return guess;
- } else {
- return Double(guess).NextDouble();
- }
-}
-
-float Strtof(Vector<const char> buffer, int exponent) {
- char copy_buffer[kMaxSignificantDecimalDigits];
- Vector<const char> trimmed;
- int updated_exponent;
- TrimAndCut(buffer, exponent, copy_buffer, kMaxSignificantDecimalDigits,
- &trimmed, &updated_exponent);
- exponent = updated_exponent;
-
- double double_guess;
- bool is_correct = ComputeGuess(trimmed, exponent, &double_guess);
-
- float float_guess = static_cast<float>(double_guess);
- if (float_guess == double_guess) {
- // This shortcut triggers for integer values.
- return float_guess;
- }
-
- // We must catch double-rounding. Say the double has been rounded up, and is
- // now a boundary of a float, and rounds up again. This is why we have to
- // look at previous too.
- // Example (in decimal numbers):
- // input: 12349
- // high-precision (4 digits): 1235
- // low-precision (3 digits):
- // when read from input: 123
- // when rounded from high precision: 124.
- // To do this we simply look at the neigbors of the correct result and see
- // if they would round to the same float. If the guess is not correct we have
- // to look at four values (since two different doubles could be the correct
- // double).
-
- double double_next = Double(double_guess).NextDouble();
- double double_previous = Double(double_guess).PreviousDouble();
-
- float f1 = static_cast<float>(double_previous);
-#ifndef NDEBUG
- float f2 = float_guess;
-#endif
- float f3 = static_cast<float>(double_next);
- float f4;
- if (is_correct) {
- f4 = f3;
- } else {
- double double_next2 = Double(double_next).NextDouble();
- f4 = static_cast<float>(double_next2);
- }
-#ifndef NDEBUG
- ASSERT(f1 <= f2 && f2 <= f3 && f3 <= f4);
-#endif
-
- // If the guess doesn't lie near a single-precision boundary we can simply
- // return its float-value.
- if (f1 == f4) {
- return float_guess;
- }
-
- ASSERT((f1 != f2 && f2 == f3 && f3 == f4) ||
- (f1 == f2 && f2 != f3 && f3 == f4) ||
- (f1 == f2 && f2 == f3 && f3 != f4));
-
- // guess and next are the two possible canditates (in the same way that
- // double_guess was the lower candidate for a double-precision guess).
- float guess = f1;
- float next = f4;
- DiyFp upper_boundary;
- if (guess == 0.0f) {
- float min_float = 1e-45f;
- upper_boundary = Double(static_cast<double>(min_float) / 2).AsDiyFp();
- } else {
- upper_boundary = Single(guess).UpperBoundary();
- }
- int comparison = CompareBufferWithDiyFp(trimmed, exponent, upper_boundary);
- if (comparison < 0) {
- return guess;
- } else if (comparison > 0) {
- return next;
- } else if ((Single(guess).Significand() & 1) == 0) {
- // Round towards even.
- return guess;
- } else {
- return next;
- }
-}
-
-} // namespace double_conversion
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/double-conversion/strtod.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/double-conversion/strtod.h b/ext/kenlm/util/double-conversion/strtod.h
deleted file mode 100644
index ed0293b..0000000
--- a/ext/kenlm/util/double-conversion/strtod.h
+++ /dev/null
@@ -1,45 +0,0 @@
-// Copyright 2010 the V8 project authors. All rights reserved.
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following
-// disclaimer in the documentation and/or other materials provided
-// with the distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived
-// from this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#ifndef DOUBLE_CONVERSION_STRTOD_H_
-#define DOUBLE_CONVERSION_STRTOD_H_
-
-#include "utils.h"
-
-namespace double_conversion {
-
-// The buffer must only contain digits in the range [0-9]. It must not
-// contain a dot or a sign. It must not start with '0', and must not be empty.
-double Strtod(Vector<const char> buffer, int exponent);
-
-// The buffer must only contain digits in the range [0-9]. It must not
-// contain a dot or a sign. It must not start with '0', and must not be empty.
-float Strtof(Vector<const char> buffer, int exponent);
-
-} // namespace double_conversion
-
-#endif // DOUBLE_CONVERSION_STRTOD_H_
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/double-conversion/utils.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/double-conversion/utils.h b/ext/kenlm/util/double-conversion/utils.h
deleted file mode 100644
index 9ccb3b6..0000000
--- a/ext/kenlm/util/double-conversion/utils.h
+++ /dev/null
@@ -1,320 +0,0 @@
-// Copyright 2010 the V8 project authors. All rights reserved.
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following
-// disclaimer in the documentation and/or other materials provided
-// with the distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived
-// from this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#ifndef DOUBLE_CONVERSION_UTILS_H_
-#define DOUBLE_CONVERSION_UTILS_H_
-
-#include <stdlib.h>
-#include <string.h>
-
-#include <assert.h>
-#ifndef ASSERT
-#define ASSERT(condition) (assert(condition))
-#endif
-#ifndef UNIMPLEMENTED
-#define UNIMPLEMENTED() (abort())
-#endif
-#ifndef UNREACHABLE
-#define UNREACHABLE() (abort())
-#endif
-
-// Double operations detection based on target architecture.
-// Linux uses a 80bit wide floating point stack on x86. This induces double
-// rounding, which in turn leads to wrong results.
-// An easy way to test if the floating-point operations are correct is to
-// evaluate: 89255.0/1e22. If the floating-point stack is 64 bits wide then
-// the result is equal to 89255e-22.
-// The best way to test this, is to create a division-function and to compare
-// the output of the division with the expected result. (Inlining must be
-// disabled.)
-// On Linux,x86 89255e-22 != Div_double(89255.0/1e22)
-#if defined(_M_X64) || defined(__x86_64__) || \
- defined(__ARMEL__) || defined(__avr32__) || \
- defined(__hppa__) || defined(__ia64__) || \
- defined(__mips__) || defined(__powerpc__) || \
- defined(__sparc__) || defined(__sparc) || defined(__s390__) || \
- defined(__SH4__) || defined(__alpha__) || \
- defined(_MIPS_ARCH_MIPS32R2)
-#define DOUBLE_CONVERSION_CORRECT_DOUBLE_OPERATIONS 1
-#elif defined(_M_IX86) || defined(__i386__) || defined(__i386)
-#if defined(_WIN32)
-// Windows uses a 64bit wide floating point stack.
-#define DOUBLE_CONVERSION_CORRECT_DOUBLE_OPERATIONS 1
-#else
-#undef DOUBLE_CONVERSION_CORRECT_DOUBLE_OPERATIONS
-#endif // _WIN32
-#else
-#error Target architecture was not detected as supported by Double-Conversion.
-#endif
-
-
-#if defined(_WIN32) && !defined(__MINGW32__)
-
-typedef signed char int8_t;
-typedef unsigned char uint8_t;
-typedef short int16_t; // NOLINT
-typedef unsigned short uint16_t; // NOLINT
-typedef int int32_t;
-typedef unsigned int uint32_t;
-typedef __int64 int64_t;
-typedef unsigned __int64 uint64_t;
-// intptr_t and friends are defined in crtdefs.h through stdio.h.
-
-#else
-
-#include <stdint.h>
-
-#endif
-
-// The following macro works on both 32 and 64-bit platforms.
-// Usage: instead of writing 0x1234567890123456
-// write UINT64_2PART_C(0x12345678,90123456);
-#define UINT64_2PART_C(a, b) (((static_cast<uint64_t>(a) << 32) + 0x##b##u))
-
-
-// The expression ARRAY_SIZE(a) is a compile-time constant of type
-// size_t which represents the number of elements of the given
-// array. You should only use ARRAY_SIZE on statically allocated
-// arrays.
-#ifndef ARRAY_SIZE
-#define ARRAY_SIZE(a) \
- ((sizeof(a) / sizeof(*(a))) / \
- static_cast<size_t>(!(sizeof(a) % sizeof(*(a)))))
-#endif
-
-// A macro to disallow the evil copy constructor and operator= functions
-// This should be used in the private: declarations for a class
-#ifndef DISALLOW_COPY_AND_ASSIGN
-#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
- TypeName(const TypeName&); \
- void operator=(const TypeName&)
-#endif
-
-// A macro to disallow all the implicit constructors, namely the
-// default constructor, copy constructor and operator= functions.
-//
-// This should be used in the private: declarations for a class
-// that wants to prevent anyone from instantiating it. This is
-// especially useful for classes containing only static methods.
-#ifndef DISALLOW_IMPLICIT_CONSTRUCTORS
-#define DISALLOW_IMPLICIT_CONSTRUCTORS(TypeName) \
- TypeName(); \
- DISALLOW_COPY_AND_ASSIGN(TypeName)
-#endif
-
-namespace double_conversion {
-
-static const int kCharSize = sizeof(char);
-
-// Returns the maximum of the two parameters.
-template <typename T>
-static T Max(T a, T b) {
- return a < b ? b : a;
-}
-
-
-// Returns the minimum of the two parameters.
-template <typename T>
-static T Min(T a, T b) {
- return a < b ? a : b;
-}
-
-
-inline int StrLength(const char* string) {
- size_t length = strlen(string);
- ASSERT(length == static_cast<size_t>(static_cast<int>(length)));
- return static_cast<int>(length);
-}
-
-// This is a simplified version of V8's Vector class.
-template <typename T>
-class Vector {
- public:
- Vector() : start_(NULL), length_(0) {}
- Vector(T* data, int length) : start_(data), length_(length) {
- ASSERT(length == 0 || (length > 0 && data != NULL));
- }
-
- // Returns a vector using the same backing storage as this one,
- // spanning from and including 'from', to but not including 'to'.
- Vector<T> SubVector(int from, int to) {
- ASSERT(to <= length_);
- ASSERT(from < to);
- ASSERT(0 <= from);
- return Vector<T>(start() + from, to - from);
- }
-
- // Returns the length of the vector.
- int length() const { return length_; }
-
- // Returns whether or not the vector is empty.
- bool is_empty() const { return length_ == 0; }
-
- // Returns the pointer to the start of the data in the vector.
- T* start() const { return start_; }
-
- // Access individual vector elements - checks bounds in debug mode.
- T& operator[](int index) const {
- ASSERT(0 <= index && index < length_);
- return start_[index];
- }
-
- T& first() { return start_[0]; }
-
- T& last() { return start_[length_ - 1]; }
-
- private:
- T* start_;
- int length_;
-};
-
-
-// Helper class for building result strings in a character buffer. The
-// purpose of the class is to use safe operations that checks the
-// buffer bounds on all operations in debug mode.
-class StringBuilder {
- public:
- StringBuilder(char* buffer, int size)
- : buffer_(buffer, size), position_(0) { }
-
- ~StringBuilder() { if (!is_finalized()) Finalize(); }
-
- int size() const { return buffer_.length(); }
-
- // Get the current position in the builder.
- int position() const {
- ASSERT(!is_finalized());
- return position_;
- }
-
- // Reset the position.
- void Reset() { position_ = 0; }
-
- // Add a single character to the builder. It is not allowed to add
- // 0-characters; use the Finalize() method to terminate the string
- // instead.
- void AddCharacter(char c) {
- // I just extract raw data not a cstr so null is fine.
- //ASSERT(c != '\0');
- ASSERT(!is_finalized() && position_ < buffer_.length());
- buffer_[position_++] = c;
- }
-
- // Add an entire string to the builder. Uses strlen() internally to
- // compute the length of the input string.
- void AddString(const char* s) {
- AddSubstring(s, StrLength(s));
- }
-
- // Add the first 'n' characters of the given string 's' to the
- // builder. The input string must have enough characters.
- void AddSubstring(const char* s, int n) {
- ASSERT(!is_finalized() && position_ + n < buffer_.length());
- // I just extract raw data not a cstr so null is fine.
- //ASSERT(static_cast<size_t>(n) <= strlen(s));
- memmove(&buffer_[position_], s, n * kCharSize);
- position_ += n;
- }
-
-
- // Add character padding to the builder. If count is non-positive,
- // nothing is added to the builder.
- void AddPadding(char c, int count) {
- for (int i = 0; i < count; i++) {
- AddCharacter(c);
- }
- }
-
- // Finalize the string by 0-terminating it and returning the buffer.
- char* Finalize() {
- ASSERT(!is_finalized() && position_ < buffer_.length());
- buffer_[position_] = '\0';
- // Make sure nobody managed to add a 0-character to the
- // buffer while building the string.
- // I just extract raw data not a cstr so null is fine.
- //ASSERT(strlen(buffer_.start()) == static_cast<size_t>(position_));
- position_ = -1;
- ASSERT(is_finalized());
- return buffer_.start();
- }
-
- private:
- Vector<char> buffer_;
- int position_;
-
- bool is_finalized() const { return position_ < 0; }
-
- DISALLOW_IMPLICIT_CONSTRUCTORS(StringBuilder);
-};
-
-// The type-based aliasing rule allows the compiler to assume that pointers of
-// different types (for some definition of different) never alias each other.
-// Thus the following code does not work:
-//
-// float f = foo();
-// int fbits = *(int*)(&f);
-//
-// The compiler 'knows' that the int pointer can't refer to f since the types
-// don't match, so the compiler may cache f in a register, leaving random data
-// in fbits. Using C++ style casts makes no difference, however a pointer to
-// char data is assumed to alias any other pointer. This is the 'memcpy
-// exception'.
-//
-// Bit_cast uses the memcpy exception to move the bits from a variable of one
-// type of a variable of another type. Of course the end result is likely to
-// be implementation dependent. Most compilers (gcc-4.2 and MSVC 2005)
-// will completely optimize BitCast away.
-//
-// There is an additional use for BitCast.
-// Recent gccs will warn when they see casts that may result in breakage due to
-// the type-based aliasing rule. If you have checked that there is no breakage
-// you can use BitCast to cast one pointer type to another. This confuses gcc
-// enough that it can no longer see that you have cast one pointer type to
-// another thus avoiding the warning.
-template <class Dest, class Source>
-inline Dest BitCast(const Source& source) {
- // Compile time assertion: sizeof(Dest) == sizeof(Source)
- // A compile error here means your Dest and Source have different sizes.
- typedef char VerifySizesAreEqual[sizeof(Dest) == sizeof(Source) ? 1 : -1]
-#if __GNUC__ > 4 || __GNUC__ == 4 && __GNUC_MINOR__ >= 8
- __attribute__((unused))
-#endif
- ;
-
- Dest dest;
- memmove(&dest, &source, sizeof(dest));
- return dest;
-}
-
-template <class Dest, class Source>
-inline Dest BitCast(Source* source) {
- return BitCast<Dest>(reinterpret_cast<uintptr_t>(source));
-}
-
-} // namespace double_conversion
-
-#endif // DOUBLE_CONVERSION_UTILS_H_
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/ersatz_progress.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/ersatz_progress.cc b/ext/kenlm/util/ersatz_progress.cc
deleted file mode 100644
index 55c82e7..0000000
--- a/ext/kenlm/util/ersatz_progress.cc
+++ /dev/null
@@ -1,47 +0,0 @@
-#include "util/ersatz_progress.hh"
-
-#include <algorithm>
-#include <ostream>
-#include <limits>
-#include <string>
-
-namespace util {
-
-namespace { const unsigned char kWidth = 100; }
-
-const char kProgressBanner[] = "----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100\n";
-
-ErsatzProgress::ErsatzProgress() : current_(0), next_(std::numeric_limits<uint64_t>::max()), complete_(next_), out_(NULL) {}
-
-ErsatzProgress::~ErsatzProgress() {
- if (out_) Finished();
-}
-
-ErsatzProgress::ErsatzProgress(uint64_t complete, std::ostream *to, const std::string &message)
- : current_(0), next_(complete / kWidth), complete_(complete), stones_written_(0), out_(to) {
- if (!out_) {
- next_ = std::numeric_limits<uint64_t>::max();
- return;
- }
- if (!message.empty()) *out_ << message << '\n';
- *out_ << kProgressBanner;
-}
-
-void ErsatzProgress::Milestone() {
- if (!out_) { current_ = 0; return; }
- if (!complete_) return;
- unsigned char stone = std::min(static_cast<uint64_t>(kWidth), (current_ * kWidth) / complete_);
-
- for (; stones_written_ < stone; ++stones_written_) {
- (*out_) << '*';
- }
- if (stone == kWidth) {
- (*out_) << std::endl;
- next_ = std::numeric_limits<uint64_t>::max();
- out_ = NULL;
- } else {
- next_ = std::max(next_, ((stone + 1) * complete_ + kWidth - 1) / kWidth);
- }
-}
-
-} // namespace util
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/ersatz_progress.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/ersatz_progress.hh b/ext/kenlm/util/ersatz_progress.hh
deleted file mode 100644
index b47aded..0000000
--- a/ext/kenlm/util/ersatz_progress.hh
+++ /dev/null
@@ -1,57 +0,0 @@
-#ifndef UTIL_ERSATZ_PROGRESS_H
-#define UTIL_ERSATZ_PROGRESS_H
-
-#include <iostream>
-#include <string>
-#include <stdint.h>
-
-// Ersatz version of boost::progress so core language model doesn't depend on
-// boost. Also adds option to print nothing.
-
-namespace util {
-
-extern const char kProgressBanner[];
-
-class ErsatzProgress {
- public:
- // No output.
- ErsatzProgress();
-
- // Null means no output. The null value is useful for passing along the ostream pointer from another caller.
- explicit ErsatzProgress(uint64_t complete, std::ostream *to = &std::cerr, const std::string &message = "");
-
- ~ErsatzProgress();
-
- ErsatzProgress &operator++() {
- if (++current_ >= next_) Milestone();
- return *this;
- }
-
- ErsatzProgress &operator+=(uint64_t amount) {
- if ((current_ += amount) >= next_) Milestone();
- return *this;
- }
-
- void Set(uint64_t to) {
- if ((current_ = to) >= next_) Milestone();
- }
-
- void Finished() {
- Set(complete_);
- }
-
- private:
- void Milestone();
-
- uint64_t current_, next_, complete_;
- unsigned char stones_written_;
- std::ostream *out_;
-
- // noncopyable
- ErsatzProgress(const ErsatzProgress &other);
- ErsatzProgress &operator=(const ErsatzProgress &other);
-};
-
-} // namespace util
-
-#endif // UTIL_ERSATZ_PROGRESS_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/exception.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/exception.cc b/ext/kenlm/util/exception.cc
deleted file mode 100644
index e644d2c..0000000
--- a/ext/kenlm/util/exception.cc
+++ /dev/null
@@ -1,105 +0,0 @@
-#include "util/exception.hh"
-
-#ifdef __GXX_RTTI
-#include <typeinfo>
-#endif
-
-#include <cerrno>
-#include <cstring>
-
-#if defined(_WIN32) || defined(_WIN64)
-#include <windows.h>
-#include <io.h>
-#endif
-
-namespace util {
-
-Exception::Exception() throw() {}
-Exception::~Exception() throw() {}
-
-void Exception::SetLocation(const char *file, unsigned int line, const char *func, const char *child_name, const char *condition) {
- /* The child class might have set some text, but we want this to come first.
- * Another option would be passing this information to the constructor, but
- * then child classes would have to accept constructor arguments and pass
- * them down.
- */
- std::string old_text;
- std::swap(old_text, what_);
- StringStream stream(what_);
- stream << file << ':' << line;
- if (func) stream << " in " << func << " threw ";
- if (child_name) {
- stream << child_name;
- } else {
-#ifdef __GXX_RTTI
- stream << typeid(this).name();
-#else
- stream << "an exception";
-#endif
- }
- if (condition) {
- stream << " because `" << condition << '\'';
- }
- stream << ".\n";
- stream << old_text;
-}
-
-namespace {
-
-#ifdef __GNUC__
-const char *HandleStrerror(int ret, const char *buf) __attribute__ ((unused));
-const char *HandleStrerror(const char *ret, const char * /*buf*/) __attribute__ ((unused));
-#endif
-// At least one of these functions will not be called.
-#ifdef __clang__
-#pragma clang diagnostic push
-#pragma clang diagnostic ignored "-Wunused-function"
-#endif
-// The XOPEN version.
-const char *HandleStrerror(int ret, const char *buf) {
- if (!ret) return buf;
- return NULL;
-}
-
-// The GNU version.
-const char *HandleStrerror(const char *ret, const char * /*buf*/) {
- return ret;
-}
-#ifdef __clang__
-#pragma clang diagnostic pop
-#endif
-} // namespace
-
-ErrnoException::ErrnoException() throw() : errno_(errno) {
- char buf[200];
- buf[0] = 0;
-#if defined(sun) || defined(_WIN32) || defined(_WIN64)
- const char *add = strerror(errno);
-#else
- const char *add = HandleStrerror(strerror_r(errno, buf, 200), buf);
-#endif
-
- if (add) {
- *this << add << ' ';
- }
-}
-
-ErrnoException::~ErrnoException() throw() {}
-
-OverflowException::OverflowException() throw() {}
-OverflowException::~OverflowException() throw() {}
-
-#if defined(_WIN32) || defined(_WIN64)
-WindowsException::WindowsException() throw() {
- unsigned int last_error = GetLastError();
- char error_msg[256] = "";
- if (!FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, last_error, LANG_NEUTRAL, error_msg, sizeof(error_msg), NULL)) {
- *this << "Windows error " << GetLastError() << " while formatting Windows error " << last_error << ". ";
- } else {
- *this << "Windows error " << last_error << ": " << error_msg;
- }
-}
-WindowsException::~WindowsException() throw() {}
-#endif
-
-} // namespace util
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/exception.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/exception.hh b/ext/kenlm/util/exception.hh
deleted file mode 100644
index 57d803d..0000000
--- a/ext/kenlm/util/exception.hh
+++ /dev/null
@@ -1,159 +0,0 @@
-#ifndef UTIL_EXCEPTION_H
-#define UTIL_EXCEPTION_H
-
-#include "util/string_stream.hh"
-
-#include <exception>
-#include <limits>
-#include <string>
-#include <stdint.h>
-
-namespace util {
-
-template <class Except, class Data> typename Except::template ExceptionTag<Except&>::Identity operator<<(Except &e, const Data &data);
-
-class Exception : public std::exception {
- public:
- Exception() throw();
- virtual ~Exception() throw();
-
- const char *what() const throw() { return what_.c_str(); }
-
- // For use by the UTIL_THROW macros.
- void SetLocation(
- const char *file,
- unsigned int line,
- const char *func,
- const char *child_name,
- const char *condition);
-
- private:
- template <class Except, class Data> friend typename Except::template ExceptionTag<Except&>::Identity operator<<(Except &e, const Data &data);
-
- // This helps restrict operator<< defined below.
- template <class T> struct ExceptionTag {
- typedef T Identity;
- };
-
- std::string what_;
-};
-
-/* This implements the normal operator<< for Exception and all its children.
- * SFINAE means it only applies to Exception. Think of this as an ersatz
- * boost::enable_if.
- */
-template <class Except, class Data> typename Except::template ExceptionTag<Except&>::Identity operator<<(Except &e, const Data &data) {
- StringStream(e.what_) << data;
- return e;
-}
-
-#ifdef __GNUC__
-#define UTIL_FUNC_NAME __PRETTY_FUNCTION__
-#else
-#ifdef _WIN32
-#define UTIL_FUNC_NAME __FUNCTION__
-#else
-#define UTIL_FUNC_NAME NULL
-#endif
-#endif
-
-/* Create an instance of Exception, add the message Modify, and throw it.
- * Modify is appended to the what() message and can contain << for ostream
- * operations.
- *
- * do .. while kludge to swallow trailing ; character
- * http://gcc.gnu.org/onlinedocs/cpp/Swallowing-the-Semicolon.html .
- * Arg can be a constructor argument to the exception.
- */
-#define UTIL_THROW_BACKEND(Condition, Exception, Arg, Modify) do { \
- Exception UTIL_e Arg; \
- UTIL_e.SetLocation(__FILE__, __LINE__, UTIL_FUNC_NAME, #Exception, Condition); \
- UTIL_e << Modify; \
- throw UTIL_e; \
-} while (0)
-
-#define UTIL_THROW_ARG(Exception, Arg, Modify) \
- UTIL_THROW_BACKEND(NULL, Exception, Arg, Modify)
-
-#define UTIL_THROW(Exception, Modify) \
- UTIL_THROW_BACKEND(NULL, Exception, , Modify);
-
-#define UTIL_THROW2(Modify) \
- UTIL_THROW_BACKEND(NULL, util::Exception, , Modify);
-
-#if __GNUC__ >= 3
-#define UTIL_UNLIKELY(x) __builtin_expect (!!(x), 0)
-#else
-#define UTIL_UNLIKELY(x) (x)
-#endif
-
-#if __GNUC__ >= 3
-#define UTIL_LIKELY(x) __builtin_expect (!!(x), 1)
-#else
-#define UTIL_LIKELY(x) (x)
-#endif
-
-#define UTIL_THROW_IF_ARG(Condition, Exception, Arg, Modify) do { \
- if (UTIL_UNLIKELY(Condition)) { \
- UTIL_THROW_BACKEND(#Condition, Exception, Arg, Modify); \
- } \
-} while (0)
-
-#define UTIL_THROW_IF(Condition, Exception, Modify) \
- UTIL_THROW_IF_ARG(Condition, Exception, , Modify)
-
-#define UTIL_THROW_IF2(Condition, Modify) \
- UTIL_THROW_IF_ARG(Condition, util::Exception, , Modify)
-
-// Exception that records errno and adds it to the message.
-class ErrnoException : public Exception {
- public:
- ErrnoException() throw();
-
- virtual ~ErrnoException() throw();
-
- int Error() const throw() { return errno_; }
-
- private:
- int errno_;
-};
-
-// file wasn't there, or couldn't be open for some reason
-class FileOpenException : public Exception {
- public:
- FileOpenException() throw() {}
- ~FileOpenException() throw() {}
-};
-
-// Utilities for overflow checking.
-class OverflowException : public Exception {
- public:
- OverflowException() throw();
- ~OverflowException() throw();
-};
-
-template <unsigned len> inline std::size_t CheckOverflowInternal(uint64_t value) {
- UTIL_THROW_IF(value > static_cast<uint64_t>(std::numeric_limits<std::size_t>::max()), OverflowException, "Integer overflow detected. This model is too big for 32-bit code.");
- return value;
-}
-
-template <> inline std::size_t CheckOverflowInternal<8>(uint64_t value) {
- return value;
-}
-
-inline std::size_t CheckOverflow(uint64_t value) {
- return CheckOverflowInternal<sizeof(std::size_t)>(value);
-}
-
-#if defined(_WIN32) || defined(_WIN64)
-/* Thrown for Windows specific operations. */
-class WindowsException : public Exception {
- public:
- WindowsException() throw();
- ~WindowsException() throw();
-};
-#endif
-
-} // namespace util
-
-#endif // UTIL_EXCEPTION_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/fake_ostream.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/fake_ostream.hh b/ext/kenlm/util/fake_ostream.hh
deleted file mode 100644
index 2f76053..0000000
--- a/ext/kenlm/util/fake_ostream.hh
+++ /dev/null
@@ -1,111 +0,0 @@
-#ifndef UTIL_FAKE_OSTREAM_H
-#define UTIL_FAKE_OSTREAM_H
-
-#include "util/float_to_string.hh"
-#include "util/integer_to_string.hh"
-#include "util/string_piece.hh"
-
-#include <cassert>
-#include <limits>
-
-#include <stdint.h>
-
-namespace util {
-
-/* Like std::ostream but without being incredibly slow.
- * Supports most of the built-in types except for long double.
- *
- * The FakeOStream class is intended to be inherited from. The inherting class
- * should provide:
- * public:
- * Derived &flush();
- * Derived &write(const void *data, std::size_t length);
- *
- * private: or protected:
- * friend class FakeOStream;
- * char *Ensure(std::size_t amount);
- * void AdvanceTo(char *to);
- *
- * The Ensure function makes enough space for an in-place write and returns
- * where to write. The AdvanceTo function happens after the write, saying how
- * much was actually written.
- *
- * Precondition:
- * amount <= kToStringMaxBytes for in-place writes.
- */
-template <class Derived> class FakeOStream {
- public:
- FakeOStream() {}
-
- // This also covers std::string and char*
- Derived &operator<<(StringPiece str) {
- return C().write(str.data(), str.size());
- }
-
- // Handle integers by size and signedness.
- private:
- template <class Arg> struct EnableIfKludge {
- typedef Derived type;
- };
- template <class From, unsigned Length = sizeof(From), bool Signed = std::numeric_limits<From>::is_signed, bool IsInteger = std::numeric_limits<From>::is_integer> struct Coerce {};
-
- template <class From> struct Coerce<From, 2, false, true> { typedef uint16_t To; };
- template <class From> struct Coerce<From, 4, false, true> { typedef uint32_t To; };
- template <class From> struct Coerce<From, 8, false, true> { typedef uint64_t To; };
-
- template <class From> struct Coerce<From, 2, true, true> { typedef int16_t To; };
- template <class From> struct Coerce<From, 4, true, true> { typedef int32_t To; };
- template <class From> struct Coerce<From, 8, true, true> { typedef int64_t To; };
- public:
- template <class From> typename EnableIfKludge<typename Coerce<From>::To>::type &operator<<(const From value) {
- return CallToString(static_cast<typename Coerce<From>::To>(value));
- }
-
- // Character types that get copied as bytes instead of displayed as integers.
- Derived &operator<<(char val) { return put(val); }
- Derived &operator<<(signed char val) { return put(static_cast<char>(val)); }
- Derived &operator<<(unsigned char val) { return put(static_cast<char>(val)); }
-
- Derived &operator<<(bool val) { return put(val + '0'); }
- // enums will fall back to int but are not caught by the template.
- Derived &operator<<(int val) { return CallToString(static_cast<typename Coerce<int>::To>(val)); }
-
- Derived &operator<<(float val) { return CallToString(val); }
- Derived &operator<<(double val) { return CallToString(val); }
-
- // This is here to catch all the other pointer types.
- Derived &operator<<(const void *value) { return CallToString(value); }
- // This is here because the above line also catches const char*.
- Derived &operator<<(const char *value) { return *this << StringPiece(value); }
- Derived &operator<<(char *value) { return *this << StringPiece(value); }
-
- Derived &put(char val) {
- char *c = C().Ensure(1);
- *c = val;
- C().AdvanceTo(++c);
- return C();
- }
-
- char widen(char val) const { return val; }
-
- private:
- // References to derived class for convenience.
- Derived &C() {
- return *static_cast<Derived*>(this);
- }
-
- const Derived &C() const {
- return *static_cast<const Derived*>(this);
- }
-
- // This is separate to prevent an infinite loop if the compiler considers
- // types the same (i.e. gcc std::size_t and uint64_t or uint32_t).
- template <class T> Derived &CallToString(const T value) {
- C().AdvanceTo(ToString(value, C().Ensure(ToStringBuf<T>::kBytes)));
- return C();
- }
-};
-
-} // namespace
-
-#endif // UTIL_FAKE_OSTREAM_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/file.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/file.cc b/ext/kenlm/util/file.cc
deleted file mode 100644
index e8976bc..0000000
--- a/ext/kenlm/util/file.cc
+++ /dev/null
@@ -1,574 +0,0 @@
-#define _LARGEFILE64_SOURCE
-#define _FILE_OFFSET_BITS 64
-
-#include "util/file.hh"
-
-#include "util/exception.hh"
-
-#include <algorithm>
-#include <cstdlib>
-#include <cstdio>
-#include <iostream>
-#include <limits>
-#include <sstream>
-
-
-#include <cassert>
-#include <cerrno>
-#include <climits>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <stdint.h>
-
-#if defined(__MINGW32__)
-#include <windows.h>
-#include <unistd.h>
-#warning "The file functions on MinGW have not been tested for file sizes above 2^31 - 1. Please read https://stackoverflow.com/questions/12539488/determine-64-bit-file-size-in-c-on-mingw-32-bit and fix"
-#elif defined(_WIN32) || defined(_WIN64)
-#include <windows.h>
-#include <io.h>
-#else
-#include <unistd.h>
-#endif
-
-namespace util {
-
-scoped_fd::~scoped_fd() {
- if (fd_ != -1 && close(fd_)) {
- std::cerr << "Could not close file " << fd_ << std::endl;
- std::abort();
- }
-}
-
-void scoped_FILE_closer::Close(std::FILE *file) {
- if (file && std::fclose(file)) {
- std::cerr << "Could not close file " << file << std::endl;
- std::abort();
- }
-}
-
-// Note that ErrnoException records errno before NameFromFD is called.
-FDException::FDException(int fd) throw() : fd_(fd), name_guess_(NameFromFD(fd)) {
- *this << "in " << name_guess_ << ' ';
-}
-
-FDException::~FDException() throw() {}
-
-EndOfFileException::EndOfFileException() throw() {
- *this << "End of file";
-}
-EndOfFileException::~EndOfFileException() throw() {}
-
-bool InputFileIsStdin(StringPiece path) {
- return path == "-" || path == "/dev/stdin";
-}
-
-bool OutputFileIsStdout(StringPiece path) {
- return path == "-" || path == "/dev/stdout";
-}
-
-int OpenReadOrThrow(const char *name) {
- int ret;
-#if defined(_WIN32) || defined(_WIN64)
- UTIL_THROW_IF(-1 == (ret = _open(name, _O_BINARY | _O_RDONLY)), ErrnoException, "while opening " << name);
-#else
- UTIL_THROW_IF(-1 == (ret = open(name, O_RDONLY)), ErrnoException, "while opening " << name);
-#endif
- return ret;
-}
-
-int CreateOrThrow(const char *name) {
- int ret;
-#if defined(_WIN32) || defined(_WIN64)
- UTIL_THROW_IF(-1 == (ret = _open(name, _O_CREAT | _O_TRUNC | _O_RDWR | _O_BINARY, _S_IREAD | _S_IWRITE)), ErrnoException, "while creating " << name);
-#else
- UTIL_THROW_IF(-1 == (ret = open(name, O_CREAT | O_TRUNC | O_RDWR, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)), ErrnoException, "while creating " << name);
-#endif
- return ret;
-}
-
-uint64_t SizeFile(int fd) {
-#if defined __MINGW32__
- struct stat sb;
- // Does this handle 64-bit?
- int ret = fstat(fd, &sb);
- if (ret == -1 || (!sb.st_size && !S_ISREG(sb.st_mode))) return kBadSize;
- return sb.st_size;
-#elif defined(_WIN32) || defined(_WIN64)
- __int64 ret = _filelengthi64(fd);
- return (ret == -1) ? kBadSize : ret;
-#else // Not windows.
-
-#ifdef OS_ANDROID
- struct stat64 sb;
- int ret = fstat64(fd, &sb);
-#else
- struct stat sb;
- int ret = fstat(fd, &sb);
-#endif
- if (ret == -1 || (!sb.st_size && !S_ISREG(sb.st_mode))) return kBadSize;
- return sb.st_size;
-#endif
-}
-
-uint64_t SizeOrThrow(int fd) {
- uint64_t ret = SizeFile(fd);
- UTIL_THROW_IF_ARG(ret == kBadSize, FDException, (fd), "Failed to size");
- return ret;
-}
-
-void ResizeOrThrow(int fd, uint64_t to) {
-#if defined __MINGW32__
- // Does this handle 64-bit?
- int ret = ftruncate
-#elif defined(_WIN32) || defined(_WIN64)
- errno_t ret = _chsize_s
-#elif defined(OS_ANDROID)
- int ret = ftruncate64
-#else
- int ret = ftruncate
-#endif
- (fd, to);
- UTIL_THROW_IF_ARG(ret, FDException, (fd), "while resizing to " << to << " bytes");
-}
-
-namespace {
-std::size_t GuardLarge(std::size_t size) {
- // The following operating systems have broken read/write/pread/pwrite that
- // only supports up to 2^31.
- // OS X man pages claim to support 64-bit, but Kareem M. Darwish had problems
- // building with larger files, so APPLE is also here.
-#if defined(_WIN32) || defined(_WIN64) || defined(__APPLE__) || defined(OS_ANDROID) || defined(__MINGW32__)
- return size < INT_MAX ? size : INT_MAX;
-#else
- return size;
-#endif
-}
-}
-
-#if defined(_WIN32) || defined(_WIN64)
-namespace {
-const std::size_t kMaxDWORD = static_cast<std::size_t>(4294967295UL);
-} // namespace
-#endif
-
-std::size_t PartialRead(int fd, void *to, std::size_t amount) {
-#if defined(_WIN32) || defined(_WIN64)
- DWORD ret;
- HANDLE file_handle = reinterpret_cast<HANDLE>(_get_osfhandle(fd));
- DWORD larger_size = static_cast<DWORD>(std::min<std::size_t>(kMaxDWORD, amount));
- DWORD smaller_size = 28672; // Received reports that 31346 worked but higher values did not. This rounds down to the nearest multiple of 4096, the page size.
- if (!ReadFile(file_handle, to, larger_size, &ret, NULL))
- {
- DWORD last_error = GetLastError();
- if (last_error != ERROR_NOT_ENOUGH_MEMORY || !ReadFile(file_handle, to, smaller_size, &ret, NULL)) {
- UTIL_THROW(WindowsException, "Windows error in ReadFile.");
- }
- }
-#else
- errno = 0;
- ssize_t ret;
- do {
- ret = read(fd, to, GuardLarge(amount));
- } while (ret == -1 && errno == EINTR);
- UTIL_THROW_IF_ARG(ret < 0, FDException, (fd), "while reading " << amount << " bytes");
-#endif
- return static_cast<std::size_t>(ret);
-}
-
-void ReadOrThrow(int fd, void *to_void, std::size_t amount) {
- uint8_t *to = static_cast<uint8_t*>(to_void);
- while (amount) {
- std::size_t ret = PartialRead(fd, to, amount);
- UTIL_THROW_IF(ret == 0, EndOfFileException, " in " << NameFromFD(fd) << " but there should be " << amount << " more bytes to read.");
- amount -= ret;
- to += ret;
- }
-}
-
-std::size_t ReadOrEOF(int fd, void *to_void, std::size_t amount) {
- uint8_t *to = static_cast<uint8_t*>(to_void);
- std::size_t remaining = amount;
- while (remaining) {
- std::size_t ret = PartialRead(fd, to, remaining);
- if (!ret) return amount - remaining;
- remaining -= ret;
- to += ret;
- }
- return amount;
-}
-
-void WriteOrThrow(int fd, const void *data_void, std::size_t size) {
- const uint8_t *data = static_cast<const uint8_t*>(data_void);
- while (size) {
-#if defined(_WIN32) || defined(_WIN64)
- int ret;
-#else
- ssize_t ret;
-#endif
- errno = 0;
- do {
- ret =
-#if defined(_WIN32) || defined(_WIN64)
- _write
-#else
- write
-#endif
- (fd, data, GuardLarge(size));
- } while (ret == -1 && errno == EINTR);
- UTIL_THROW_IF_ARG(ret < 1, FDException, (fd), "while writing " << size << " bytes");
- data += ret;
- size -= ret;
- }
-}
-
-void WriteOrThrow(FILE *to, const void *data, std::size_t size) {
- if (!size) return;
- UTIL_THROW_IF(1 != std::fwrite(data, size, 1, to), ErrnoException, "Short write; requested size " << size);
-}
-
-void ErsatzPRead(int fd, void *to_void, std::size_t size, uint64_t off) {
- uint8_t *to = static_cast<uint8_t*>(to_void);
- while (size) {
-#if defined(_WIN32) || defined(_WIN64)
- /* BROKEN: changes file pointer. Even if you save it and change it back, it won't be safe to use concurrently with write() or read() which lmplz does. */
- // size_t might be 64-bit. DWORD is always 32.
- DWORD reading = static_cast<DWORD>(std::min<std::size_t>(kMaxDWORD, size));
- DWORD ret;
- OVERLAPPED overlapped;
- memset(&overlapped, 0, sizeof(OVERLAPPED));
- overlapped.Offset = static_cast<DWORD>(off);
- overlapped.OffsetHigh = static_cast<DWORD>(off >> 32);
- UTIL_THROW_IF(!ReadFile((HANDLE)_get_osfhandle(fd), to, reading, &ret, &overlapped), WindowsException, "ReadFile failed for offset " << off);
-#else
- ssize_t ret;
- errno = 0;
- ret =
-#ifdef OS_ANDROID
- pread64
-#else
- pread
-#endif
- (fd, to, GuardLarge(size), off);
- if (ret <= 0) {
- if (ret == -1 && errno == EINTR) continue;
- UTIL_THROW_IF(ret == 0, EndOfFileException, " for reading " << size << " bytes at " << off << " from " << NameFromFD(fd));
- UTIL_THROW_ARG(FDException, (fd), "while reading " << size << " bytes at offset " << off);
- }
-#endif
- size -= ret;
- off += ret;
- to += ret;
- }
-}
-
-void ErsatzPWrite(int fd, const void *from_void, std::size_t size, uint64_t off) {
- const uint8_t *from = static_cast<const uint8_t*>(from_void);
- while(size) {
-#if defined(_WIN32) || defined(_WIN64)
- /* Changes file pointer. Even if you save it and change it back, it won't be safe to use concurrently with write() or read() */
- // size_t might be 64-bit. DWORD is always 32.
- DWORD writing = static_cast<DWORD>(std::min<std::size_t>(kMaxDWORD, size));
- DWORD ret;
- OVERLAPPED overlapped;
- memset(&overlapped, 0, sizeof(OVERLAPPED));
- overlapped.Offset = static_cast<DWORD>(off);
- overlapped.OffsetHigh = static_cast<DWORD>(off >> 32);
- UTIL_THROW_IF(!WriteFile((HANDLE)_get_osfhandle(fd), from, writing, &ret, &overlapped), Exception, "WriteFile failed for offset " << off);
-#else
- ssize_t ret;
- errno = 0;
- ret =
-#ifdef OS_ANDROID
- pwrite64
-#else
- pwrite
-#endif
- (fd, from, GuardLarge(size), off);
- if (ret <= 0) {
- if (ret == -1 && errno == EINTR) continue;
- UTIL_THROW_IF(ret == 0, EndOfFileException, " for writing " << size << " bytes at " << off << " from " << NameFromFD(fd));
- UTIL_THROW_ARG(FDException, (fd), "while writing " << size << " bytes at offset " << off);
- }
-#endif
- size -= ret;
- off += ret;
- from += ret;
- }
-}
-
-
-void FSyncOrThrow(int fd) {
-// Apparently windows doesn't have fsync?
-#if !defined(_WIN32) && !defined(_WIN64)
- UTIL_THROW_IF_ARG(-1 == fsync(fd), FDException, (fd), "while syncing");
-#endif
-}
-
-namespace {
-
-// Static assert for 64-bit off_t size.
-#if !defined(_WIN32) && !defined(_WIN64) && !defined(OS_ANDROID)
-template <unsigned> struct CheckOffT;
-template <> struct CheckOffT<8> {
- struct True {};
-};
-// If there's a compiler error on the next line, then off_t isn't 64 bit. And
-// that makes me a sad panda.
-typedef CheckOffT<sizeof(off_t)>::True IgnoredType;
-#endif
-
-// Can't we all just get along?
-void InternalSeek(int fd, int64_t off, int whence) {
- if (
-#if defined __MINGW32__
- // Does this handle 64-bit?
- (off_t)-1 == lseek(fd, off, whence)
-#elif defined(_WIN32) || defined(_WIN64)
- (__int64)-1 == _lseeki64(fd, off, whence)
-#elif defined(OS_ANDROID)
- (off64_t)-1 == lseek64(fd, off, whence)
-#else
- (off_t)-1 == lseek(fd, off, whence)
-#endif
- ) UTIL_THROW_ARG(FDException, (fd), "while seeking to " << off << " whence " << whence);
-}
-} // namespace
-
-void SeekOrThrow(int fd, uint64_t off) {
- InternalSeek(fd, off, SEEK_SET);
-}
-
-void AdvanceOrThrow(int fd, int64_t off) {
- InternalSeek(fd, off, SEEK_CUR);
-}
-
-void SeekEnd(int fd) {
- InternalSeek(fd, 0, SEEK_END);
-}
-
-std::FILE *FDOpenOrThrow(scoped_fd &file) {
- std::FILE *ret = fdopen(file.get(), "r+b");
- UTIL_THROW_IF_ARG(!ret, FDException, (file.get()), "Could not fdopen for write");
- file.release();
- return ret;
-}
-
-std::FILE *FDOpenReadOrThrow(scoped_fd &file) {
- std::FILE *ret = fdopen(file.get(), "rb");
- UTIL_THROW_IF_ARG(!ret, FDException, (file.get()), "Could not fdopen for read");
- file.release();
- return ret;
-}
-
-// Sigh. Windows temporary file creation is full of race conditions.
-#if defined(_WIN32) || defined(_WIN64)
-/* mkstemp extracted from libc/sysdeps/posix/tempname.c. Copyright
- (C) 1991-1999, 2000, 2001, 2006 Free Software Foundation, Inc.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version. */
-
-/* This has been modified from the original version to rename the function and
- * set the Windows temporary flag. */
-
-static const char letters[] =
-"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
-
-/* Generate a temporary file name based on TMPL. TMPL must match the
- rules for mk[s]temp (i.e. end in "XXXXXX"). The name constructed
- does not exist at the time of the call to mkstemp. TMPL is
- overwritten with the result. */
-int
-mkstemp_and_unlink(char *tmpl)
-{
- int len;
- char *XXXXXX;
- static unsigned long long value;
- unsigned long long random_time_bits;
- unsigned int count;
- int fd = -1;
- int save_errno = errno;
-
- /* A lower bound on the number of temporary files to attempt to
- generate. The maximum total number of temporary file names that
- can exist for a given template is 62**6. It should never be
- necessary to try all these combinations. Instead if a reasonable
- number of names is tried (we define reasonable as 62**3) fail to
- give the system administrator the chance to remove the problems. */
-#define ATTEMPTS_MIN (62 * 62 * 62)
-
- /* The number of times to attempt to generate a temporary file. To
- conform to POSIX, this must be no smaller than TMP_MAX. */
-#if ATTEMPTS_MIN < TMP_MAX
- unsigned int attempts = TMP_MAX;
-#else
- unsigned int attempts = ATTEMPTS_MIN;
-#endif
-
- len = strlen (tmpl);
- if (len < 6 || strcmp (&tmpl[len - 6], "XXXXXX"))
- {
- errno = EINVAL;
- return -1;
- }
-
-/* This is where the Xs start. */
- XXXXXX = &tmpl[len - 6];
-
- /* Get some more or less random data. */
- {
- SYSTEMTIME stNow;
- FILETIME ftNow;
-
- // get system time
- GetSystemTime(&stNow);
- stNow.wMilliseconds = 500;
- if (!SystemTimeToFileTime(&stNow, &ftNow))
- {
- errno = -1;
- return -1;
- }
-
- random_time_bits = (((unsigned long long)ftNow.dwHighDateTime << 32)
- | (unsigned long long)ftNow.dwLowDateTime);
- }
- value += random_time_bits ^ (unsigned long long)GetCurrentThreadId ();
-
- for (count = 0; count < attempts; value += 7777, ++count)
- {
- unsigned long long v = value;
-
- /* Fill in the random bits. */
- XXXXXX[0] = letters[v % 62];
- v /= 62;
- XXXXXX[1] = letters[v % 62];
- v /= 62;
- XXXXXX[2] = letters[v % 62];
- v /= 62;
- XXXXXX[3] = letters[v % 62];
- v /= 62;
- XXXXXX[4] = letters[v % 62];
- v /= 62;
- XXXXXX[5] = letters[v % 62];
-
- /* Modified for windows and to unlink */
- // fd = open (tmpl, O_RDWR | O_CREAT | O_EXCL, _S_IREAD | _S_IWRITE);
- int flags = _O_RDWR | _O_CREAT | _O_EXCL | _O_BINARY;
- flags |= _O_TEMPORARY;
- fd = _open (tmpl, flags, _S_IREAD | _S_IWRITE);
- if (fd >= 0)
- {
- errno = save_errno;
- return fd;
- }
- else if (errno != EEXIST)
- return -1;
- }
-
- /* We got out of the loop because we ran out of combinations to try. */
- errno = EEXIST;
- return -1;
-}
-#else
-int
-mkstemp_and_unlink(char *tmpl) {
- int ret = mkstemp(tmpl);
- if (ret != -1) {
- UTIL_THROW_IF(unlink(tmpl), ErrnoException, "while deleting delete " << tmpl);
- }
- return ret;
-}
-#endif
-
-// If it's a directory, add a /. This lets users say -T /tmp without creating
-// /tmpAAAAAA
-void NormalizeTempPrefix(std::string &base) {
- if (base.empty()) return;
- if (base[base.size() - 1] == '/') return;
- struct stat sb;
- // It's fine for it to not exist.
- if (-1 == stat(base.c_str(), &sb)) return;
- if (
-#if defined(_WIN32) || defined(_WIN64)
- sb.st_mode & _S_IFDIR
-#else
- S_ISDIR(sb.st_mode)
-#endif
- ) base += '/';
-}
-
-int MakeTemp(const StringPiece &base) {
- std::string name(base.data(), base.size());
- name += "XXXXXX";
- name.push_back(0);
- int ret;
- UTIL_THROW_IF(-1 == (ret = mkstemp_and_unlink(&name[0])), ErrnoException, "while making a temporary based on " << base);
- return ret;
-}
-
-std::FILE *FMakeTemp(const StringPiece &base) {
- util::scoped_fd file(MakeTemp(base));
- return FDOpenOrThrow(file);
-}
-
-int DupOrThrow(int fd) {
- int ret = dup(fd);
- UTIL_THROW_IF_ARG(ret == -1, FDException, (fd), "in duplicating the file descriptor");
- return ret;
-}
-
-namespace {
-// Try to name things but be willing to fail too.
-bool TryName(int fd, std::string &out) {
-#if defined(_WIN32) || defined(_WIN64)
- return false;
-#else
- std::string name("/proc/self/fd/");
- std::ostringstream convert;
- convert << fd;
- name += convert.str();
-
- struct stat sb;
- if (-1 == lstat(name.c_str(), &sb))
- return false;
- out.resize(sb.st_size + 1);
- // lstat gave us a size, but I've seen it grow, possibly due to symlinks on top of symlinks.
- while (true) {
- ssize_t ret = readlink(name.c_str(), &out[0], out.size());
- if (-1 == ret)
- return false;
- if ((size_t)ret < out.size()) {
- out.resize(ret);
- break;
- }
- // Exponential growth.
- out.resize(out.size() * 2);
- }
- // Don't use the non-file names.
- if (!out.empty() && out[0] != '/')
- return false;
- return true;
-#endif
-}
-} // namespace
-
-std::string NameFromFD(int fd) {
- std::string ret;
- if (TryName(fd, ret)) return ret;
- switch (fd) {
- case 0: return "stdin";
- case 1: return "stdout";
- case 2: return "stderr";
- }
- ret = "fd ";
- std::ostringstream convert;
- convert << fd;
- ret += convert.str();
- return ret;
-}
-
-} // namespace util
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/file.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/file.hh b/ext/kenlm/util/file.hh
deleted file mode 100644
index f7cb4d6..0000000
--- a/ext/kenlm/util/file.hh
+++ /dev/null
@@ -1,154 +0,0 @@
-#ifndef UTIL_FILE_H
-#define UTIL_FILE_H
-
-#include "util/exception.hh"
-#include "util/scoped.hh"
-#include "util/string_piece.hh"
-
-#include <cstddef>
-#include <cstdio>
-#include <string>
-#include <stdint.h>
-
-namespace util {
-
-class scoped_fd {
- public:
- scoped_fd() : fd_(-1) {}
-
- explicit scoped_fd(int fd) : fd_(fd) {}
-
- ~scoped_fd();
-
- void reset(int to = -1) {
- scoped_fd other(fd_);
- fd_ = to;
- }
-
- int get() const { return fd_; }
-
- int operator*() const { return fd_; }
-
- int release() {
- int ret = fd_;
- fd_ = -1;
- return ret;
- }
-
- private:
- int fd_;
-
- scoped_fd(const scoped_fd &);
- scoped_fd &operator=(const scoped_fd &);
-};
-
-struct scoped_FILE_closer {
- static void Close(std::FILE *file);
-};
-typedef scoped<std::FILE, scoped_FILE_closer> scoped_FILE;
-
-/* Thrown for any operation where the fd is known. */
-class FDException : public ErrnoException {
- public:
- explicit FDException(int fd) throw();
-
- virtual ~FDException() throw();
-
- // This may no longer be valid if the exception was thrown past open.
- int FD() const { return fd_; }
-
- // Guess from NameFromFD.
- const std::string &NameGuess() const { return name_guess_; }
-
- private:
- int fd_;
-
- std::string name_guess_;
-};
-
-// End of file reached.
-class EndOfFileException : public Exception {
- public:
- EndOfFileException() throw();
- ~EndOfFileException() throw();
-};
-
-// Open for read only.
-int OpenReadOrThrow(const char *name);
-// Create file if it doesn't exist, truncate if it does. Opened for write.
-int CreateOrThrow(const char *name);
-
-/** Does the given input file path denote standard input?
- *
- * Returns true if, and only if, path is either "-" or "/dev/stdin".
- *
- * Opening standard input as a file may need some special treatment for
- * portability. There's a convention that a dash ("-") in place of an input
- * file path denotes standard input, but opening "/dev/stdin" may need to be
- * special as well.
- */
-bool InputPathIsStdin(StringPiece path);
-
-/** Does the given output file path denote standard output?
- *
- * Returns true if, and only if, path is either "-" or "/dev/stdout".
- *
- * Opening standard output as a file may need some special treatment for
- * portability. There's a convention that a dash ("-") in place of an output
- * file path denotes standard output, but opening "/dev/stdout" may need to be
- * special as well.
- */
-bool OutputPathIsStdout(StringPiece path);
-
-// Return value for SizeFile when it can't size properly.
-const uint64_t kBadSize = (uint64_t)-1;
-uint64_t SizeFile(int fd);
-uint64_t SizeOrThrow(int fd);
-
-void ResizeOrThrow(int fd, uint64_t to);
-
-std::size_t PartialRead(int fd, void *to, std::size_t size);
-void ReadOrThrow(int fd, void *to, std::size_t size);
-std::size_t ReadOrEOF(int fd, void *to_void, std::size_t size);
-
-void WriteOrThrow(int fd, const void *data_void, std::size_t size);
-void WriteOrThrow(FILE *to, const void *data, std::size_t size);
-
-/* These call pread/pwrite in a loop. However, on Windows they call ReadFile/
- * WriteFile which changes the file pointer. So it's safe to call ErsatzPRead
- * and ErsatzPWrite concurrently (or any combination thereof). But it changes
- * the file pointer on windows, so it's not safe to call concurrently with
- * anything that uses the implicit file pointer e.g. the Read/Write functions
- * above.
- */
-void ErsatzPRead(int fd, void *to, std::size_t size, uint64_t off);
-void ErsatzPWrite(int fd, const void *data_void, std::size_t size, uint64_t off);
-
-void FSyncOrThrow(int fd);
-
-// Seeking
-void SeekOrThrow(int fd, uint64_t off);
-void AdvanceOrThrow(int fd, int64_t off);
-void SeekEnd(int fd);
-
-std::FILE *FDOpenOrThrow(scoped_fd &file);
-std::FILE *FDOpenReadOrThrow(scoped_fd &file);
-
-// Temporary files
-// Append a / if base is a directory.
-void NormalizeTempPrefix(std::string &base);
-int MakeTemp(const StringPiece &prefix);
-std::FILE *FMakeTemp(const StringPiece &prefix);
-
-// dup an fd.
-int DupOrThrow(int fd);
-
-/* Attempt get file name from fd. This won't always work (i.e. on Windows or
- * a pipe). The file might have been renamed. It's intended for diagnostics
- * and logging only.
- */
-std::string NameFromFD(int fd);
-
-} // namespace util
-
-#endif // UTIL_FILE_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/file_piece.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/file_piece.cc b/ext/kenlm/util/file_piece.cc
deleted file mode 100644
index 0a4d3a9..0000000
--- a/ext/kenlm/util/file_piece.cc
+++ /dev/null
@@ -1,337 +0,0 @@
-#include "util/file_piece.hh"
-
-#include "util/double-conversion/double-conversion.h"
-#include "util/exception.hh"
-#include "util/file.hh"
-#include "util/mmap.hh"
-
-#if defined(_WIN32) || defined(_WIN64)
-#include <io.h>
-#else
-#include <unistd.h>
-#endif
-
-#include <cassert>
-#include <cerrno>
-#include <cmath>
-#include <cstdlib>
-#include <iostream>
-#include <limits>
-#include <string>
-
-#include <fcntl.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-
-namespace util {
-
-ParseNumberException::ParseNumberException(StringPiece value) throw() {
- *this << "Could not parse \"" << value << "\" into a ";
-}
-
-// Sigh this is the only way I could come up with to do a _const_ bool. It has ' ', '\f', '\n', '\r', '\t', and '\v' (same as isspace on C locale).
-const bool kSpaces[256] = {0,0,0,0,0,0,0,0,0,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
-
-FilePiece::FilePiece(const char *name, std::ostream *show_progress, std::size_t min_buffer) :
- file_(OpenReadOrThrow(name)), total_size_(SizeFile(file_.get())), page_(SizePage()),
- progress_(total_size_, total_size_ == kBadSize ? NULL : show_progress, std::string("Reading ") + name) {
- Initialize(name, show_progress, min_buffer);
-}
-
-namespace {
-std::string NamePossiblyFind(int fd, const char *name) {
- if (name) return name;
- return NameFromFD(fd);
-}
-} // namespace
-
-FilePiece::FilePiece(int fd, const char *name, std::ostream *show_progress, std::size_t min_buffer) :
- file_(fd), total_size_(SizeFile(file_.get())), page_(SizePage()),
- progress_(total_size_, total_size_ == kBadSize ? NULL : show_progress, std::string("Reading ") + NamePossiblyFind(fd, name)) {
- Initialize(NamePossiblyFind(fd, name).c_str(), show_progress, min_buffer);
-}
-
-FilePiece::FilePiece(std::istream &stream, const char *name, std::size_t min_buffer) :
- total_size_(kBadSize), page_(SizePage()) {
- InitializeNoRead("istream", min_buffer);
-
- fallback_to_read_ = true;
- HugeMalloc(default_map_size_, false, data_);
- position_ = data_.begin();
- position_end_ = position_;
-
- fell_back_.Reset(stream);
-}
-
-FilePiece::~FilePiece() {}
-
-StringPiece FilePiece::ReadLine(char delim, bool strip_cr) {
- std::size_t skip = 0;
- while (true) {
- for (const char *i = position_ + skip; i < position_end_; ++i) {
- if (*i == delim) {
- // End of line.
- // Take 1 byte off the end if it's an unwanted carriage return.
- const std::size_t subtract_cr = (
- (strip_cr && i > position_ && *(i - 1) == '\r') ?
- 1 : 0);
- StringPiece ret(position_, i - position_ - subtract_cr);
- position_ = i + 1;
- return ret;
- }
- }
- if (at_end_) {
- if (position_ == position_end_) {
- Shift();
- }
- return Consume(position_end_);
- }
- skip = position_end_ - position_;
- Shift();
- }
-}
-
-bool FilePiece::ReadLineOrEOF(StringPiece &to, char delim, bool strip_cr) {
- try {
- to = ReadLine(delim, strip_cr);
- } catch (const util::EndOfFileException &e) { return false; }
- return true;
-}
-
-float FilePiece::ReadFloat() {
- return ReadNumber<float>();
-}
-double FilePiece::ReadDouble() {
- return ReadNumber<double>();
-}
-long int FilePiece::ReadLong() {
- return ReadNumber<long int>();
-}
-unsigned long int FilePiece::ReadULong() {
- return ReadNumber<unsigned long int>();
-}
-
-// Factored out so that istream can call this.
-void FilePiece::InitializeNoRead(const char *name, std::size_t min_buffer) {
- file_name_ = name;
-
- default_map_size_ = page_ * std::max<std::size_t>((min_buffer / page_ + 1), 2);
- position_ = NULL;
- position_end_ = NULL;
- mapped_offset_ = 0;
- at_end_ = false;
-}
-
-void FilePiece::Initialize(const char *name, std::ostream *show_progress, std::size_t min_buffer) {
- InitializeNoRead(name, min_buffer);
-
- if (total_size_ == kBadSize) {
- // So the assertion passes.
- fallback_to_read_ = false;
- if (show_progress)
- *show_progress << "File " << name << " isn't normal. Using slower read() instead of mmap(). No progress bar." << std::endl;
- TransitionToRead();
- } else {
- fallback_to_read_ = false;
- }
- Shift();
- // gzip detect.
- if ((position_end_ >= position_ + ReadCompressed::kMagicSize) && ReadCompressed::DetectCompressedMagic(position_)) {
- if (!fallback_to_read_) {
- at_end_ = false;
- TransitionToRead();
- }
- }
-}
-
-namespace {
-
-static const double_conversion::StringToDoubleConverter kConverter(
- double_conversion::StringToDoubleConverter::ALLOW_TRAILING_JUNK | double_conversion::StringToDoubleConverter::ALLOW_LEADING_SPACES,
- std::numeric_limits<double>::quiet_NaN(),
- std::numeric_limits<double>::quiet_NaN(),
- "inf",
- "NaN");
-
-StringPiece FirstToken(StringPiece str) {
- const char *i;
- for (i = str.data(); i != str.data() + str.size(); ++i) {
- if (kSpaces[(unsigned char)*i]) break;
- }
- return StringPiece(str.data(), i - str.data());
-}
-
-const char *ParseNumber(StringPiece str, float &out) {
- int count;
- out = kConverter.StringToFloat(str.data(), str.size(), &count);
- UTIL_THROW_IF_ARG(std::isnan(out) && str != "NaN" && str != "nan", ParseNumberException, (FirstToken(str)), "float");
- return str.data() + count;
-}
-const char *ParseNumber(StringPiece str, double &out) {
- int count;
- out = kConverter.StringToDouble(str.data(), str.size(), &count);
- UTIL_THROW_IF_ARG(std::isnan(out) && str != "NaN" && str != "nan", ParseNumberException, (FirstToken(str)), "double");
- return str.data() + count;
-}
-const char *ParseNumber(StringPiece str, long int &out) {
- char *end;
- errno = 0;
- out = strtol(str.data(), &end, 10);
- UTIL_THROW_IF_ARG(errno || (end == str.data()), ParseNumberException, (FirstToken(str)), "long int");
- return end;
-}
-const char *ParseNumber(StringPiece str, unsigned long int &out) {
- char *end;
- errno = 0;
- out = strtoul(str.data(), &end, 10);
- UTIL_THROW_IF_ARG(errno || (end == str.data()), ParseNumberException, (FirstToken(str)), "unsigned long int");
- return end;
-}
-} // namespace
-
-template <class T> T FilePiece::ReadNumber() {
- SkipSpaces();
- while (last_space_ < position_) {
- if (UTIL_UNLIKELY(at_end_)) {
- // Hallucinate a null off the end of the file.
- std::string buffer(position_, position_end_);
- T ret;
- // Has to be null-terminated.
- const char *begin = buffer.c_str();
- const char *end = ParseNumber(StringPiece(begin, buffer.size()), ret);
- position_ += end - begin;
- return ret;
- }
- Shift();
- }
- T ret;
- position_ = ParseNumber(StringPiece(position_, last_space_ - position_), ret);
- return ret;
-}
-
-const char *FilePiece::FindDelimiterOrEOF(const bool *delim) {
- std::size_t skip = 0;
- while (true) {
- for (const char *i = position_ + skip; i < position_end_; ++i) {
- if (delim[static_cast<unsigned char>(*i)]) return i;
- }
- if (at_end_) {
- if (position_ == position_end_) Shift();
- return position_end_;
- }
- skip = position_end_ - position_;
- Shift();
- }
-}
-
-void FilePiece::Shift() {
- if (at_end_) {
- progress_.Finished();
- throw EndOfFileException();
- }
- uint64_t desired_begin = position_ - data_.begin() + mapped_offset_;
-
- if (!fallback_to_read_) MMapShift(desired_begin);
- // Notice an mmap failure might set the fallback.
- if (fallback_to_read_) ReadShift();
-
- for (last_space_ = position_end_ - 1; last_space_ >= position_; --last_space_) {
- if (kSpaces[static_cast<unsigned char>(*last_space_)]) break;
- }
-}
-
-void FilePiece::MMapShift(uint64_t desired_begin) {
- // Use mmap.
- uint64_t ignore = desired_begin % page_;
- // Duplicate request for Shift means give more data.
- if (position_ == data_.begin() + ignore && position_) {
- default_map_size_ *= 2;
- }
- // Local version so that in case of failure it doesn't overwrite the class variable.
- uint64_t mapped_offset = desired_begin - ignore;
-
- uint64_t mapped_size;
- if (default_map_size_ >= static_cast<std::size_t>(total_size_ - mapped_offset)) {
- at_end_ = true;
- mapped_size = total_size_ - mapped_offset;
- } else {
- mapped_size = default_map_size_;
- }
-
- // Forcibly clear the existing mmap first.
- data_.reset();
- try {
- MapRead(POPULATE_OR_LAZY, *file_, mapped_offset, mapped_size, data_);
- } catch (const util::ErrnoException &e) {
- if (desired_begin) {
- SeekOrThrow(*file_, desired_begin);
- }
- // The mmap was scheduled to end the file, but now we're going to read it.
- at_end_ = false;
- TransitionToRead();
- return;
- }
- mapped_offset_ = mapped_offset;
- position_ = data_.begin() + ignore;
- position_end_ = data_.begin() + mapped_size;
-
- progress_.Set(desired_begin);
-}
-
-void FilePiece::TransitionToRead() {
- assert(!fallback_to_read_);
- fallback_to_read_ = true;
- data_.reset();
- HugeMalloc(default_map_size_, false, data_);
- position_ = data_.begin();
- position_end_ = position_;
-
- try {
- fell_back_.Reset(file_.release());
- } catch (util::Exception &e) {
- e << " in file " << file_name_;
- throw;
- }
-}
-
-void FilePiece::ReadShift() {
- assert(fallback_to_read_);
- // Bytes [data_.begin(), position_) have been consumed.
- // Bytes [position_, position_end_) have been read into the buffer.
-
- // Start at the beginning of the buffer if there's nothing useful in it.
- if (position_ == position_end_) {
- mapped_offset_ += (position_end_ - data_.begin());
- position_ = data_.begin();
- position_end_ = position_;
- }
-
- std::size_t already_read = position_end_ - data_.begin();
-
- if (already_read == default_map_size_) {
- if (position_ == data_.begin()) {
- // Buffer too small.
- std::size_t valid_length = position_end_ - position_;
- default_map_size_ *= 2;
- HugeRealloc(default_map_size_, false, data_);
- position_ = data_.begin();
- position_end_ = position_ + valid_length;
- } else {
- std::size_t moving = position_end_ - position_;
- memmove(data_.get(), position_, moving);
- position_ = data_.begin();
- position_end_ = position_ + moving;
- already_read = moving;
- }
- }
-
- std::size_t read_return = fell_back_.Read(static_cast<uint8_t*>(data_.get()) + already_read, default_map_size_ - already_read);
- progress_.Set(fell_back_.RawAmount());
-
- if (read_return == 0) {
- at_end_ = true;
- }
- position_end_ += read_return;
-}
-
-} // namespace util
[10/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/tune_instance_data/toy1.2
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/tune_instance_data/toy1.2 b/ext/kenlm/lm/interpolate/tune_instance_data/toy1.2
deleted file mode 100644
index 58d28a0..0000000
Binary files a/ext/kenlm/lm/interpolate/tune_instance_data/toy1.2 and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/tune_instance_data/toy1.3
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/tune_instance_data/toy1.3 b/ext/kenlm/lm/interpolate/tune_instance_data/toy1.3
deleted file mode 100644
index 1a63afe..0000000
Binary files a/ext/kenlm/lm/interpolate/tune_instance_data/toy1.3 and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/tune_instance_data/toy1.kenlm_intermediate
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/tune_instance_data/toy1.kenlm_intermediate b/ext/kenlm/lm/interpolate/tune_instance_data/toy1.kenlm_intermediate
deleted file mode 100644
index fe82667..0000000
--- a/ext/kenlm/lm/interpolate/tune_instance_data/toy1.kenlm_intermediate
+++ /dev/null
@@ -1,3 +0,0 @@
-KenLM intermediate binary file
-Counts 6 7 6
-Payload pb
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/tune_instance_data/toy1.vocab
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/tune_instance_data/toy1.vocab b/ext/kenlm/lm/interpolate/tune_instance_data/toy1.vocab
deleted file mode 100644
index 763b2af..0000000
Binary files a/ext/kenlm/lm/interpolate/tune_instance_data/toy1.vocab and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/tune_instance_test.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/tune_instance_test.cc b/ext/kenlm/lm/interpolate/tune_instance_test.cc
deleted file mode 100644
index a0db59c..0000000
--- a/ext/kenlm/lm/interpolate/tune_instance_test.cc
+++ /dev/null
@@ -1,93 +0,0 @@
-#include "lm/interpolate/tune_instance.hh"
-
-#include "util/file_stream.hh"
-#include "util/file.hh"
-#include "util/string_piece.hh"
-
-#define BOOST_TEST_MODULE InstanceTest
-#include <boost/test/unit_test.hpp>
-
-#include <iostream>
-
-#include <vector>
-
-namespace lm { namespace interpolate { namespace {
-
-Matrix::Index FindRow(const std::vector<WordIndex> &words, WordIndex word) {
- std::vector<WordIndex>::const_iterator it = std::find(words.begin(), words.end(), word);
- BOOST_REQUIRE(it != words.end());
- return it - words.begin();
-}
-
-BOOST_AUTO_TEST_CASE(Toy) {
- util::scoped_fd test_input(util::MakeTemp("temporary"));
- {
- util::FileStream(test_input.get()) << "c\n";
- }
-
- StringPiece dir("tune_instance_data/");
- if (boost::unit_test::framework::master_test_suite().argc == 2) {
- StringPiece zero_file(boost::unit_test::framework::master_test_suite().argv[1]);
- BOOST_REQUIRE(zero_file.size() > strlen("toy0.1"));
- BOOST_REQUIRE_EQUAL("toy0.1", StringPiece(zero_file.data() + zero_file.size() - 6, 6));
- dir = StringPiece(zero_file.data(), zero_file.size() - 6);
- }
-
- std::vector<StringPiece> model_names;
- std::string full0 = std::string(dir.data(), dir.size()) + "toy0";
- std::string full1 = std::string(dir.data(), dir.size()) + "toy1";
- model_names.push_back(full0);
- model_names.push_back(full1);
-
- util::FixedArray<Instance> instances;
- Matrix ln_unigrams;
- // Returns vocab id of <s>
- BOOST_CHECK_EQUAL(1, LoadInstances(test_input.release(), model_names, instances, ln_unigrams));
- // <unk>
- BOOST_CHECK_CLOSE(-0.90309 * M_LN10, ln_unigrams(0, 0), 0.001);
- BOOST_CHECK_CLOSE(-1 * M_LN10, ln_unigrams(0, 1), 0.001);
- // <s>
- BOOST_CHECK_GT(-98.0, ln_unigrams(1, 0));
- BOOST_CHECK_GT(-98.0, ln_unigrams(1, 1));
- // a
- BOOST_CHECK_CLOSE(-0.46943438 * M_LN10, ln_unigrams(2, 0), 0.001);
- BOOST_CHECK_CLOSE(-0.6146491 * M_LN10, ln_unigrams(2, 1), 0.001);
- // </s>
- BOOST_CHECK_CLOSE(-0.5720968 * M_LN10, ln_unigrams(3, 0), 0.001);
- BOOST_CHECK_CLOSE(-0.6146491 * M_LN10, ln_unigrams(3, 1), 0.001);
- // c
- BOOST_CHECK_CLOSE(-0.90309 * M_LN10, ln_unigrams(4, 0), 0.001); // <unk>
- BOOST_CHECK_CLOSE(-0.7659168 * M_LN10, ln_unigrams(4, 1), 0.001);
- // too lazy to do b.
-
- // Two instances:
- // <s> predicts c
- // <s> c predicts </s>
- BOOST_REQUIRE_EQUAL(2, instances.size());
- BOOST_CHECK_CLOSE(-0.30103 * M_LN10, instances[0].ln_backoff(0), 0.001);
- BOOST_CHECK_CLOSE(-0.30103 * M_LN10, instances[0].ln_backoff(1), 0.001);
-
- // Backoffs of <s> c
- BOOST_CHECK_CLOSE(0.0, instances[1].ln_backoff(0), 0.001);
- BOOST_CHECK_CLOSE((-0.30103 - 0.30103) * M_LN10, instances[1].ln_backoff(1), 0.001);
-
- // Three extensions: a, b, c
- BOOST_REQUIRE_EQUAL(3, instances[0].ln_extensions.rows());
- BOOST_REQUIRE_EQUAL(3, instances[0].extension_words.size());
-
- // <s> a
- BOOST_CHECK_CLOSE(-0.37712017 * M_LN10, instances[0].ln_extensions(FindRow(instances[0].extension_words, 2), 0), 0.001);
- // <s> c
- BOOST_CHECK_CLOSE((-0.90309 + -0.30103) * M_LN10, instances[0].ln_extensions(FindRow(instances[0].extension_words, 4), 0), 0.001);
- BOOST_CHECK_CLOSE(-0.4740302 * M_LN10, instances[0].ln_extensions(FindRow(instances[0].extension_words, 4), 1), 0.001);
-
- // <s> c </s>
- BOOST_CHECK_CLOSE(-0.09113217 * M_LN10, instances[1].ln_extensions(FindRow(instances[1].extension_words, 3), 1), 0.001);
-
- // p_0(c | <s>) = p_0(c)b_0(<s>) = 10^(-0.90309 + -0.30103)
- BOOST_CHECK_CLOSE((-0.90309 + -0.30103) * M_LN10, instances[0].ln_correct(0), 0.001);
- // p_1(c | <s>) = 10^-0.4740302
- BOOST_CHECK_CLOSE(-0.4740302 * M_LN10, instances[0].ln_correct(1), 0.001);
-}
-
-}}} // namespaces
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/tune_main.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/tune_main.cc b/ext/kenlm/lm/interpolate/tune_main.cc
deleted file mode 100644
index 8296af1..0000000
--- a/ext/kenlm/lm/interpolate/tune_main.cc
+++ /dev/null
@@ -1,59 +0,0 @@
-#include "lm/interpolate/tune_derivatives.hh"
-#include "lm/interpolate/tune_instance.hh"
-#include "util/file.hh"
-
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wunused-local-typedefs"
-#include <Eigen/Dense>
-#pragma GCC diagnostic pop
-#include <boost/program_options.hpp>
-
-#include <cmath>
-#include <iostream>
-
-namespace lm { namespace interpolate {
-void TuneWeights(int tune_file, const std::vector<StringPiece> &model_names, Vector &weights) {
- util::FixedArray<Instance> instances;
- Matrix ln_unigrams;
- WordIndex bos = LoadInstances(tune_file, model_names, instances, ln_unigrams);
- ComputeDerivative derive(instances, ln_unigrams, bos);
- weights = Vector::Constant(model_names.size(), 1.0 / model_names.size());
- Vector gradient;
- Matrix hessian;
- for (std::size_t iteration = 0; iteration < 10 /*TODO fancy stopping criteria */; ++iteration) {
- std::cerr << "Iteration " << iteration << ": weights =";
- for (Vector::Index i = 0; i < weights.rows(); ++i) {
- std::cerr << ' ' << weights(i);
- }
- std::cerr << std::endl;
- std::cerr << "Perplexity = " <<
- derive.Iteration(weights, gradient, hessian)
- << std::endl;
- // TODO: 1.0 step size was too big and it kept getting unstable. More math.
- weights -= 0.7 * hessian.inverse() * gradient;
- }
-}
-}} // namespaces
-
-int main(int argc, char *argv[]) {
- Eigen::initParallel();
- namespace po = boost::program_options;
- // TODO help
- po::options_description options("Tuning options");
- std::string tuning_file;
- std::vector<std::string> input_models;
- options.add_options()
- ("tuning,t", po::value<std::string>(&tuning_file)->required(), "File to tune on. This should be a text file with one sentence per line.")
- ("model,m", po::value<std::vector<std::string> >(&input_models)->multitoken()->required(), "Models to interpolate");
- po::variables_map vm;
- po::store(po::parse_command_line(argc, argv, options), vm);
- po::notify(vm);
-
- std::vector<StringPiece> model_names;
- for (std::vector<std::string>::const_iterator i = input_models.begin(); i != input_models.end(); ++i) {
- model_names.push_back(*i);
- }
- lm::interpolate::Vector weights;
- lm::interpolate::TuneWeights(util::OpenReadOrThrow(tuning_file.c_str()), model_names, weights);
- std::cout << weights.transpose() << std::endl;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/tune_matrix.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/tune_matrix.hh b/ext/kenlm/lm/interpolate/tune_matrix.hh
deleted file mode 100644
index 7f1a0c9..0000000
--- a/ext/kenlm/lm/interpolate/tune_matrix.hh
+++ /dev/null
@@ -1,17 +0,0 @@
-#ifndef LM_INTERPOLATE_TUNE_MATRIX_H
-#define LM_INTERPOLATE_TUNE_MATRIX_H
-
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wunused-local-typedefs"
-#include <Eigen/Core>
-#pragma GCC diagnostic pop
-
-namespace lm { namespace interpolate {
-
-typedef Eigen::MatrixXd Matrix;
-typedef Eigen::VectorXd Vector;
-
-typedef Matrix::Scalar Accum;
-
-}} // namespaces
-#endif // LM_INTERPOLATE_TUNE_MATRIX_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/universal_vocab.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/universal_vocab.cc b/ext/kenlm/lm/interpolate/universal_vocab.cc
deleted file mode 100644
index 5cdf41e..0000000
--- a/ext/kenlm/lm/interpolate/universal_vocab.cc
+++ /dev/null
@@ -1,13 +0,0 @@
-#include "lm/interpolate/universal_vocab.hh"
-
-namespace lm {
-namespace interpolate {
-
-UniversalVocab::UniversalVocab(const std::vector<WordIndex>& model_vocab_sizes) {
- model_index_map_.resize(model_vocab_sizes.size());
- for (size_t i = 0; i < model_vocab_sizes.size(); ++i) {
- model_index_map_[i].resize(model_vocab_sizes[i]);
- }
-}
-
-}} // namespaces
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/interpolate/universal_vocab.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/interpolate/universal_vocab.hh b/ext/kenlm/lm/interpolate/universal_vocab.hh
deleted file mode 100644
index c720298..0000000
--- a/ext/kenlm/lm/interpolate/universal_vocab.hh
+++ /dev/null
@@ -1,38 +0,0 @@
-#ifndef LM_INTERPOLATE_UNIVERSAL_VOCAB_H
-#define LM_INTERPOLATE_UNIVERSAL_VOCAB_H
-
-#include "lm/word_index.hh"
-
-#include <vector>
-#include <cstddef>
-
-namespace lm {
-namespace interpolate {
-
-class UniversalVocab {
-public:
- explicit UniversalVocab(const std::vector<WordIndex>& model_vocab_sizes);
-
- // GetUniversalIndex takes the model number and index for the specific
- // model and returns the universal model number
- WordIndex GetUniversalIdx(std::size_t model_num, WordIndex model_word_index) const {
- return model_index_map_[model_num][model_word_index];
- }
-
- const WordIndex *Mapping(std::size_t model) const {
- return &*model_index_map_[model].begin();
- }
-
- void InsertUniversalIdx(std::size_t model_num, WordIndex word_index,
- WordIndex universal_word_index) {
- model_index_map_[model_num][word_index] = universal_word_index;
- }
-
-private:
- std::vector<std::vector<WordIndex> > model_index_map_;
-};
-
-} // namespace interpolate
-} // namespace lm
-
-#endif // LM_INTERPOLATE_UNIVERSAL_VOCAB_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/kenlm_benchmark_main.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/kenlm_benchmark_main.cc b/ext/kenlm/lm/kenlm_benchmark_main.cc
deleted file mode 100644
index c9ee165..0000000
--- a/ext/kenlm/lm/kenlm_benchmark_main.cc
+++ /dev/null
@@ -1,142 +0,0 @@
-#include "lm/model.hh"
-#include "util/file_stream.hh"
-#include "util/file.hh"
-#include "util/file_piece.hh"
-#include "util/usage.hh"
-
-#include <stdint.h>
-
-namespace {
-
-template <class Model, class Width> void ConvertToBytes(const Model &model, int fd_in) {
- util::FilePiece in(fd_in);
- util::FileStream out(1);
- Width width;
- StringPiece word;
- const Width end_sentence = (Width)model.GetVocabulary().EndSentence();
- while (true) {
- while (in.ReadWordSameLine(word)) {
- width = (Width)model.GetVocabulary().Index(word);
- out.write(&width, sizeof(Width));
- }
- if (!in.ReadLineOrEOF(word)) break;
- out.write(&end_sentence, sizeof(Width));
- }
-}
-
-template <class Model, class Width> void QueryFromBytes(const Model &model, int fd_in) {
- lm::ngram::State state[3];
- const lm::ngram::State *const begin_state = &model.BeginSentenceState();
- const lm::ngram::State *next_state = begin_state;
- Width kEOS = model.GetVocabulary().EndSentence();
- Width buf[4096];
-
- uint64_t completed = 0;
- double loaded = util::CPUTime();
-
- std::cout << "CPU_to_load: " << loaded << std::endl;
-
- // Numerical precision: batch sums.
- double total = 0.0;
- while (std::size_t got = util::ReadOrEOF(fd_in, buf, sizeof(buf))) {
- float sum = 0.0;
- UTIL_THROW_IF2(got % sizeof(Width), "File size not a multiple of vocab id size " << sizeof(Width));
- got /= sizeof(Width);
- completed += got;
- // Do even stuff first.
- const Width *even_end = buf + (got & ~1);
- // Alternating states
- const Width *i;
- for (i = buf; i != even_end;) {
- sum += model.FullScore(*next_state, *i, state[1]).prob;
- next_state = (*i++ == kEOS) ? begin_state : &state[1];
- sum += model.FullScore(*next_state, *i, state[0]).prob;
- next_state = (*i++ == kEOS) ? begin_state : &state[0];
- }
- // Odd corner case.
- if (got & 1) {
- sum += model.FullScore(*next_state, *i, state[2]).prob;
- next_state = (*i++ == kEOS) ? begin_state : &state[2];
- }
- total += sum;
- }
- double after = util::CPUTime();
- std::cerr << "Probability sum is " << total << std::endl;
- std::cout << "Queries: " << completed << std::endl;
- std::cout << "CPU_excluding_load: " << (after - loaded) << "\nCPU_per_query: " << ((after - loaded) / static_cast<double>(completed)) << std::endl;
- std::cout << "RSSMax: " << util::RSSMax() << std::endl;
-}
-
-template <class Model, class Width> void DispatchFunction(const Model &model, bool query) {
- if (query) {
- QueryFromBytes<Model, Width>(model, 0);
- } else {
- ConvertToBytes<Model, Width>(model, 0);
- }
-}
-
-template <class Model> void DispatchWidth(const char *file, bool query) {
- lm::ngram::Config config;
- config.load_method = util::READ;
- std::cerr << "Using load_method = READ." << std::endl;
- Model model(file, config);
- lm::WordIndex bound = model.GetVocabulary().Bound();
- if (bound <= 256) {
- DispatchFunction<Model, uint8_t>(model, query);
- } else if (bound <= 65536) {
- DispatchFunction<Model, uint16_t>(model, query);
- } else if (bound <= (1ULL << 32)) {
- DispatchFunction<Model, uint32_t>(model, query);
- } else {
- DispatchFunction<Model, uint64_t>(model, query);
- }
-}
-
-void Dispatch(const char *file, bool query) {
- using namespace lm::ngram;
- lm::ngram::ModelType model_type;
- if (lm::ngram::RecognizeBinary(file, model_type)) {
- switch(model_type) {
- case PROBING:
- DispatchWidth<lm::ngram::ProbingModel>(file, query);
- break;
- case REST_PROBING:
- DispatchWidth<lm::ngram::RestProbingModel>(file, query);
- break;
- case TRIE:
- DispatchWidth<lm::ngram::TrieModel>(file, query);
- break;
- case QUANT_TRIE:
- DispatchWidth<lm::ngram::QuantTrieModel>(file, query);
- break;
- case ARRAY_TRIE:
- DispatchWidth<lm::ngram::ArrayTrieModel>(file, query);
- break;
- case QUANT_ARRAY_TRIE:
- DispatchWidth<lm::ngram::QuantArrayTrieModel>(file, query);
- break;
- default:
- UTIL_THROW(util::Exception, "Unrecognized kenlm model type " << model_type);
- }
- } else {
- UTIL_THROW(util::Exception, "Binarize before running benchmarks.");
- }
-}
-
-} // namespace
-
-int main(int argc, char *argv[]) {
- if (argc != 3 || (strcmp(argv[1], "vocab") && strcmp(argv[1], "query"))) {
- std::cerr
- << "Benchmark program for KenLM. Intended usage:\n"
- << "#Convert text to vocabulary ids offline. These ids are tied to a model.\n"
- << argv[0] << " vocab $model <$text >$text.vocab\n"
- << "#Ensure files are in RAM.\n"
- << "cat $text.vocab $model >/dev/null\n"
- << "#Timed query against the model.\n"
- << argv[0] << " query $model <$text.vocab\n";
- return 1;
- }
- Dispatch(argv[2], !strcmp(argv[1], "query"));
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/left.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/left.hh b/ext/kenlm/lm/left.hh
deleted file mode 100644
index 4d49686..0000000
--- a/ext/kenlm/lm/left.hh
+++ /dev/null
@@ -1,216 +0,0 @@
-/* Efficient left and right language model state for sentence fragments.
- * Intended usage:
- * Store ChartState with every chart entry.
- * To do a rule application:
- * 1. Make a ChartState object for your new entry.
- * 2. Construct RuleScore.
- * 3. Going from left to right, call Terminal or NonTerminal.
- * For terminals, just pass the vocab id.
- * For non-terminals, pass that non-terminal's ChartState.
- * If your decoder expects scores inclusive of subtree scores (i.e. you
- * label entries with the highest-scoring path), pass the non-terminal's
- * score as prob.
- * If your decoder expects relative scores and will walk the chart later,
- * pass prob = 0.0.
- * In other words, the only effect of prob is that it gets added to the
- * returned log probability.
- * 4. Call Finish. It returns the log probability.
- *
- * There's a couple more details:
- * Do not pass <s> to Terminal as it is formally not a word in the sentence,
- * only context. Instead, call BeginSentence. If called, it should be the
- * first call after RuleScore is constructed (since <s> is always the
- * leftmost).
- *
- * If the leftmost RHS is a non-terminal, it's faster to call BeginNonTerminal.
- *
- * Hashing and sorting comparison operators are provided. All state objects
- * are POD. If you intend to use memcmp on raw state objects, you must call
- * ZeroRemaining first, as the value of array entries beyond length is
- * otherwise undefined.
- *
- * Usage is of course not limited to chart decoding. Anything that generates
- * sentence fragments missing left context could benefit. For example, a
- * phrase-based decoder could pre-score phrases, storing ChartState with each
- * phrase, even if hypotheses are generated left-to-right.
- */
-
-#ifndef LM_LEFT_H
-#define LM_LEFT_H
-
-#include "lm/max_order.hh"
-#include "lm/state.hh"
-#include "lm/return.hh"
-
-#include "util/murmur_hash.hh"
-
-#include <algorithm>
-
-namespace lm {
-namespace ngram {
-
-template <class M> class RuleScore {
- public:
- explicit RuleScore(const M &model, ChartState &out) : model_(model), out_(&out), left_done_(false), prob_(0.0) {
- out.left.length = 0;
- out.right.length = 0;
- }
-
- void BeginSentence() {
- out_->right = model_.BeginSentenceState();
- // out_->left is empty.
- left_done_ = true;
- }
-
- void Terminal(WordIndex word) {
- State copy(out_->right);
- FullScoreReturn ret(model_.FullScore(copy, word, out_->right));
- if (left_done_) { prob_ += ret.prob; return; }
- if (ret.independent_left) {
- prob_ += ret.prob;
- left_done_ = true;
- return;
- }
- out_->left.pointers[out_->left.length++] = ret.extend_left;
- prob_ += ret.rest;
- if (out_->right.length != copy.length + 1)
- left_done_ = true;
- }
-
- // Faster version of NonTerminal for the case where the rule begins with a non-terminal.
- void BeginNonTerminal(const ChartState &in, float prob = 0.0) {
- prob_ = prob;
- *out_ = in;
- left_done_ = in.left.full;
- }
-
- void NonTerminal(const ChartState &in, float prob = 0.0) {
- prob_ += prob;
-
- if (!in.left.length) {
- if (in.left.full) {
- for (const float *i = out_->right.backoff; i < out_->right.backoff + out_->right.length; ++i) prob_ += *i;
- left_done_ = true;
- out_->right = in.right;
- }
- return;
- }
-
- if (!out_->right.length) {
- out_->right = in.right;
- if (left_done_) {
- prob_ += model_.UnRest(in.left.pointers, in.left.pointers + in.left.length, 1);
- return;
- }
- if (out_->left.length) {
- left_done_ = true;
- } else {
- out_->left = in.left;
- left_done_ = in.left.full;
- }
- return;
- }
-
- float backoffs[KENLM_MAX_ORDER - 1], backoffs2[KENLM_MAX_ORDER - 1];
- float *back = backoffs, *back2 = backoffs2;
- unsigned char next_use = out_->right.length;
-
- // First word
- if (ExtendLeft(in, next_use, 1, out_->right.backoff, back)) return;
-
- // Words after the first, so extending a bigram to begin with
- for (unsigned char extend_length = 2; extend_length <= in.left.length; ++extend_length) {
- if (ExtendLeft(in, next_use, extend_length, back, back2)) return;
- std::swap(back, back2);
- }
-
- if (in.left.full) {
- for (const float *i = back; i != back + next_use; ++i) prob_ += *i;
- left_done_ = true;
- out_->right = in.right;
- return;
- }
-
- // Right state was minimized, so it's already independent of the new words to the left.
- if (in.right.length < in.left.length) {
- out_->right = in.right;
- return;
- }
-
- // Shift exisiting words down.
- for (WordIndex *i = out_->right.words + next_use - 1; i >= out_->right.words; --i) {
- *(i + in.right.length) = *i;
- }
- // Add words from in.right.
- std::copy(in.right.words, in.right.words + in.right.length, out_->right.words);
- // Assemble backoff composed on the existing state's backoff followed by the new state's backoff.
- std::copy(in.right.backoff, in.right.backoff + in.right.length, out_->right.backoff);
- std::copy(back, back + next_use, out_->right.backoff + in.right.length);
- out_->right.length = in.right.length + next_use;
- }
-
- float Finish() {
- // A N-1-gram might extend left and right but we should still set full to true because it's an N-1-gram.
- out_->left.full = left_done_ || (out_->left.length == model_.Order() - 1);
- return prob_;
- }
-
- void Reset() {
- prob_ = 0.0;
- left_done_ = false;
- out_->left.length = 0;
- out_->right.length = 0;
- }
- void Reset(ChartState &replacement) {
- out_ = &replacement;
- Reset();
- }
-
- private:
- bool ExtendLeft(const ChartState &in, unsigned char &next_use, unsigned char extend_length, const float *back_in, float *back_out) {
- ProcessRet(model_.ExtendLeft(
- out_->right.words, out_->right.words + next_use, // Words to extend into
- back_in, // Backoffs to use
- in.left.pointers[extend_length - 1], extend_length, // Words to be extended
- back_out, // Backoffs for the next score
- next_use)); // Length of n-gram to use in next scoring.
- if (next_use != out_->right.length) {
- left_done_ = true;
- if (!next_use) {
- // Early exit.
- out_->right = in.right;
- prob_ += model_.UnRest(in.left.pointers + extend_length, in.left.pointers + in.left.length, extend_length + 1);
- return true;
- }
- }
- // Continue scoring.
- return false;
- }
-
- void ProcessRet(const FullScoreReturn &ret) {
- if (left_done_) {
- prob_ += ret.prob;
- return;
- }
- if (ret.independent_left) {
- prob_ += ret.prob;
- left_done_ = true;
- return;
- }
- out_->left.pointers[out_->left.length++] = ret.extend_left;
- prob_ += ret.rest;
- }
-
- const M &model_;
-
- ChartState *out_;
-
- bool left_done_;
-
- float prob_;
-};
-
-} // namespace ngram
-} // namespace lm
-
-#endif // LM_LEFT_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/left_test.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/left_test.cc b/ext/kenlm/lm/left_test.cc
deleted file mode 100644
index fdb6416..0000000
--- a/ext/kenlm/lm/left_test.cc
+++ /dev/null
@@ -1,397 +0,0 @@
-#include "lm/left.hh"
-#include "lm/model.hh"
-
-#include "util/tokenize_piece.hh"
-
-#include <vector>
-
-#define BOOST_TEST_MODULE LeftTest
-#include <boost/test/unit_test.hpp>
-#include <boost/test/floating_point_comparison.hpp>
-
-namespace lm {
-namespace ngram {
-namespace {
-
-#define Term(word) score.Terminal(m.GetVocabulary().Index(word));
-#define VCheck(word, value) BOOST_CHECK_EQUAL(m.GetVocabulary().Index(word), value);
-
-// Apparently some Boost versions use templates and are pretty strict about types matching.
-#define SLOPPY_CHECK_CLOSE(ref, value, tol) BOOST_CHECK_CLOSE(static_cast<double>(ref), static_cast<double>(value), static_cast<double>(tol));
-
-template <class M> void Short(const M &m) {
- ChartState base;
- {
- RuleScore<M> score(m, base);
- Term("more");
- Term("loin");
- SLOPPY_CHECK_CLOSE(-1.206319 - 0.3561665, score.Finish(), 0.001);
- }
- BOOST_CHECK(base.left.full);
- BOOST_CHECK_EQUAL(2, base.left.length);
- BOOST_CHECK_EQUAL(1, base.right.length);
- VCheck("loin", base.right.words[0]);
-
- ChartState more_left;
- {
- RuleScore<M> score(m, more_left);
- Term("little");
- score.NonTerminal(base, -1.206319 - 0.3561665);
- // p(little more loin | null context)
- SLOPPY_CHECK_CLOSE(-1.56538, score.Finish(), 0.001);
- }
- BOOST_CHECK_EQUAL(3, more_left.left.length);
- BOOST_CHECK_EQUAL(1, more_left.right.length);
- VCheck("loin", more_left.right.words[0]);
- BOOST_CHECK(more_left.left.full);
-
- ChartState shorter;
- {
- RuleScore<M> score(m, shorter);
- Term("to");
- score.NonTerminal(base, -1.206319 - 0.3561665);
- SLOPPY_CHECK_CLOSE(-0.30103 - 1.687872 - 1.206319 - 0.3561665, score.Finish(), 0.01);
- }
- BOOST_CHECK_EQUAL(1, shorter.left.length);
- BOOST_CHECK_EQUAL(1, shorter.right.length);
- VCheck("loin", shorter.right.words[0]);
- BOOST_CHECK(shorter.left.full);
-}
-
-template <class M> void Charge(const M &m) {
- ChartState base;
- {
- RuleScore<M> score(m, base);
- Term("on");
- Term("more");
- SLOPPY_CHECK_CLOSE(-1.509559 -0.4771212 -1.206319, score.Finish(), 0.001);
- }
- BOOST_CHECK_EQUAL(1, base.left.length);
- BOOST_CHECK_EQUAL(1, base.right.length);
- VCheck("more", base.right.words[0]);
- BOOST_CHECK(base.left.full);
-
- ChartState extend;
- {
- RuleScore<M> score(m, extend);
- Term("looking");
- score.NonTerminal(base, -1.509559 -0.4771212 -1.206319);
- SLOPPY_CHECK_CLOSE(-3.91039, score.Finish(), 0.001);
- }
- BOOST_CHECK_EQUAL(2, extend.left.length);
- BOOST_CHECK_EQUAL(1, extend.right.length);
- VCheck("more", extend.right.words[0]);
- BOOST_CHECK(extend.left.full);
-
- ChartState tobos;
- {
- RuleScore<M> score(m, tobos);
- score.BeginSentence();
- score.NonTerminal(extend, -3.91039);
- SLOPPY_CHECK_CLOSE(-3.471169, score.Finish(), 0.001);
- }
- BOOST_CHECK_EQUAL(0, tobos.left.length);
- BOOST_CHECK_EQUAL(1, tobos.right.length);
-}
-
-template <class M> float LeftToRight(const M &m, const std::vector<WordIndex> &words, bool begin_sentence = false) {
- float ret = 0.0;
- State right = begin_sentence ? m.BeginSentenceState() : m.NullContextState();
- for (std::vector<WordIndex>::const_iterator i = words.begin(); i != words.end(); ++i) {
- State copy(right);
- ret += m.Score(copy, *i, right);
- }
- return ret;
-}
-
-template <class M> float RightToLeft(const M &m, const std::vector<WordIndex> &words, bool begin_sentence = false) {
- float ret = 0.0;
- ChartState state;
- state.left.length = 0;
- state.right.length = 0;
- state.left.full = false;
- for (std::vector<WordIndex>::const_reverse_iterator i = words.rbegin(); i != words.rend(); ++i) {
- ChartState copy(state);
- RuleScore<M> score(m, state);
- score.Terminal(*i);
- score.NonTerminal(copy, ret);
- ret = score.Finish();
- }
- if (begin_sentence) {
- ChartState copy(state);
- RuleScore<M> score(m, state);
- score.BeginSentence();
- score.NonTerminal(copy, ret);
- ret = score.Finish();
- }
- return ret;
-}
-
-template <class M> float TreeMiddle(const M &m, const std::vector<WordIndex> &words, bool begin_sentence = false) {
- std::vector<std::pair<ChartState, float> > states(words.size());
- for (unsigned int i = 0; i < words.size(); ++i) {
- RuleScore<M> score(m, states[i].first);
- score.Terminal(words[i]);
- states[i].second = score.Finish();
- }
- while (states.size() > 1) {
- std::vector<std::pair<ChartState, float> > upper((states.size() + 1) / 2);
- for (unsigned int i = 0; i < states.size() / 2; ++i) {
- RuleScore<M> score(m, upper[i].first);
- score.NonTerminal(states[i*2].first, states[i*2].second);
- score.NonTerminal(states[i*2+1].first, states[i*2+1].second);
- upper[i].second = score.Finish();
- }
- if (states.size() % 2) {
- upper.back() = states.back();
- }
- std::swap(states, upper);
- }
-
- if (states.empty()) return 0.0;
-
- if (begin_sentence) {
- ChartState ignored;
- RuleScore<M> score(m, ignored);
- score.BeginSentence();
- score.NonTerminal(states.front().first, states.front().second);
- return score.Finish();
- } else {
- return states.front().second;
- }
-
-}
-
-template <class M> void LookupVocab(const M &m, const StringPiece &str, std::vector<WordIndex> &out) {
- out.clear();
- for (util::TokenIter<util::SingleCharacter, true> i(str, ' '); i; ++i) {
- out.push_back(m.GetVocabulary().Index(*i));
- }
-}
-
-#define TEXT_TEST(str) \
- LookupVocab(m, str, words); \
- expect = LeftToRight(m, words, rest); \
- SLOPPY_CHECK_CLOSE(expect, RightToLeft(m, words, rest), 0.001); \
- SLOPPY_CHECK_CLOSE(expect, TreeMiddle(m, words, rest), 0.001); \
-
-// Build sentences, or parts thereof, from right to left.
-template <class M> void GrowBig(const M &m, bool rest = false) {
- std::vector<WordIndex> words;
- float expect;
- TEXT_TEST("in biarritz watching considering looking . on a little more loin also would consider higher to look good unknown the screening foo bar , unknown however unknown </s>");
- TEXT_TEST("on a little more loin also would consider higher to look good unknown the screening foo bar , unknown however unknown </s>");
- TEXT_TEST("on a little more loin also would consider higher to look good");
- TEXT_TEST("more loin also would consider higher to look good");
- TEXT_TEST("more loin also would consider higher to look");
- TEXT_TEST("also would consider higher to look");
- TEXT_TEST("also would consider higher");
- TEXT_TEST("would consider higher to look");
- TEXT_TEST("consider higher to look");
- TEXT_TEST("consider higher to");
- TEXT_TEST("consider higher");
-}
-
-template <class M> void GrowSmall(const M &m, bool rest = false) {
- std::vector<WordIndex> words;
- float expect;
- TEXT_TEST("in biarritz watching considering looking . </s>");
- TEXT_TEST("in biarritz watching considering looking .");
- TEXT_TEST("in biarritz");
-}
-
-template <class M> void AlsoWouldConsiderHigher(const M &m) {
- ChartState also;
- {
- RuleScore<M> score(m, also);
- score.Terminal(m.GetVocabulary().Index("also"));
- SLOPPY_CHECK_CLOSE(-1.687872, score.Finish(), 0.001);
- }
- ChartState would;
- {
- RuleScore<M> score(m, would);
- score.Terminal(m.GetVocabulary().Index("would"));
- SLOPPY_CHECK_CLOSE(-1.687872, score.Finish(), 0.001);
- }
- ChartState combine_also_would;
- {
- RuleScore<M> score(m, combine_also_would);
- score.NonTerminal(also, -1.687872);
- score.NonTerminal(would, -1.687872);
- SLOPPY_CHECK_CLOSE(-1.687872 - 2.0, score.Finish(), 0.001);
- }
- BOOST_CHECK_EQUAL(2, combine_also_would.right.length);
-
- ChartState also_would;
- {
- RuleScore<M> score(m, also_would);
- score.Terminal(m.GetVocabulary().Index("also"));
- score.Terminal(m.GetVocabulary().Index("would"));
- SLOPPY_CHECK_CLOSE(-1.687872 - 2.0, score.Finish(), 0.001);
- }
- BOOST_CHECK_EQUAL(2, also_would.right.length);
-
- ChartState consider;
- {
- RuleScore<M> score(m, consider);
- score.Terminal(m.GetVocabulary().Index("consider"));
- SLOPPY_CHECK_CLOSE(-1.687872, score.Finish(), 0.001);
- }
- BOOST_CHECK_EQUAL(1, consider.left.length);
- BOOST_CHECK_EQUAL(1, consider.right.length);
- BOOST_CHECK(!consider.left.full);
-
- ChartState higher;
- float higher_score;
- {
- RuleScore<M> score(m, higher);
- score.Terminal(m.GetVocabulary().Index("higher"));
- higher_score = score.Finish();
- }
- SLOPPY_CHECK_CLOSE(-1.509559, higher_score, 0.001);
- BOOST_CHECK_EQUAL(1, higher.left.length);
- BOOST_CHECK_EQUAL(1, higher.right.length);
- BOOST_CHECK(!higher.left.full);
- VCheck("higher", higher.right.words[0]);
- SLOPPY_CHECK_CLOSE(-0.30103, higher.right.backoff[0], 0.001);
-
- ChartState consider_higher;
- {
- RuleScore<M> score(m, consider_higher);
- score.NonTerminal(consider, -1.687872);
- score.NonTerminal(higher, higher_score);
- SLOPPY_CHECK_CLOSE(-1.509559 - 1.687872 - 0.30103, score.Finish(), 0.001);
- }
- BOOST_CHECK_EQUAL(2, consider_higher.left.length);
- BOOST_CHECK(!consider_higher.left.full);
-
- ChartState full;
- {
- RuleScore<M> score(m, full);
- score.NonTerminal(combine_also_would, -1.687872 - 2.0);
- score.NonTerminal(consider_higher, -1.509559 - 1.687872 - 0.30103);
- SLOPPY_CHECK_CLOSE(-10.6879, score.Finish(), 0.001);
- }
- BOOST_CHECK_EQUAL(4, full.right.length);
-}
-
-#define CHECK_SCORE(str, val) \
-{ \
- float got = val; \
- std::vector<WordIndex> indices; \
- LookupVocab(m, str, indices); \
- SLOPPY_CHECK_CLOSE(LeftToRight(m, indices), got, 0.001); \
-}
-
-template <class M> void FullGrow(const M &m) {
- std::vector<WordIndex> words;
- LookupVocab(m, "in biarritz watching considering looking . </s>", words);
-
- ChartState lexical[7];
- float lexical_scores[7];
- for (unsigned int i = 0; i < 7; ++i) {
- RuleScore<M> score(m, lexical[i]);
- score.Terminal(words[i]);
- lexical_scores[i] = score.Finish();
- }
- CHECK_SCORE("in", lexical_scores[0]);
- CHECK_SCORE("biarritz", lexical_scores[1]);
- CHECK_SCORE("watching", lexical_scores[2]);
- CHECK_SCORE("</s>", lexical_scores[6]);
-
- ChartState l1[4];
- float l1_scores[4];
- {
- RuleScore<M> score(m, l1[0]);
- score.NonTerminal(lexical[0], lexical_scores[0]);
- score.NonTerminal(lexical[1], lexical_scores[1]);
- CHECK_SCORE("in biarritz", l1_scores[0] = score.Finish());
- }
- {
- RuleScore<M> score(m, l1[1]);
- score.NonTerminal(lexical[2], lexical_scores[2]);
- score.NonTerminal(lexical[3], lexical_scores[3]);
- CHECK_SCORE("watching considering", l1_scores[1] = score.Finish());
- }
- {
- RuleScore<M> score(m, l1[2]);
- score.NonTerminal(lexical[4], lexical_scores[4]);
- score.NonTerminal(lexical[5], lexical_scores[5]);
- CHECK_SCORE("looking .", l1_scores[2] = score.Finish());
- }
- BOOST_CHECK_EQUAL(l1[2].left.length, 1);
- l1[3] = lexical[6];
- l1_scores[3] = lexical_scores[6];
-
- ChartState l2[2];
- float l2_scores[2];
- {
- RuleScore<M> score(m, l2[0]);
- score.NonTerminal(l1[0], l1_scores[0]);
- score.NonTerminal(l1[1], l1_scores[1]);
- CHECK_SCORE("in biarritz watching considering", l2_scores[0] = score.Finish());
- }
- {
- RuleScore<M> score(m, l2[1]);
- score.NonTerminal(l1[2], l1_scores[2]);
- score.NonTerminal(l1[3], l1_scores[3]);
- CHECK_SCORE("looking . </s>", l2_scores[1] = score.Finish());
- }
- BOOST_CHECK_EQUAL(l2[1].left.length, 1);
- BOOST_CHECK(l2[1].left.full);
-
- ChartState top;
- {
- RuleScore<M> score(m, top);
- score.NonTerminal(l2[0], l2_scores[0]);
- score.NonTerminal(l2[1], l2_scores[1]);
- CHECK_SCORE("in biarritz watching considering looking . </s>", score.Finish());
- }
-}
-
-const char *FileLocation() {
- if (boost::unit_test::framework::master_test_suite().argc < 2) {
- return "test.arpa";
- }
- return boost::unit_test::framework::master_test_suite().argv[1];
-}
-
-template <class M> void Everything() {
- Config config;
- config.messages = NULL;
- M m(FileLocation(), config);
-
- Short(m);
- Charge(m);
- GrowBig(m);
- AlsoWouldConsiderHigher(m);
- GrowSmall(m);
- FullGrow(m);
-}
-
-BOOST_AUTO_TEST_CASE(ProbingAll) {
- Everything<Model>();
-}
-BOOST_AUTO_TEST_CASE(TrieAll) {
- Everything<TrieModel>();
-}
-BOOST_AUTO_TEST_CASE(QuantTrieAll) {
- Everything<QuantTrieModel>();
-}
-BOOST_AUTO_TEST_CASE(ArrayQuantTrieAll) {
- Everything<QuantArrayTrieModel>();
-}
-BOOST_AUTO_TEST_CASE(ArrayTrieAll) {
- Everything<ArrayTrieModel>();
-}
-
-BOOST_AUTO_TEST_CASE(RestProbing) {
- Config config;
- config.messages = NULL;
- RestProbingModel m(FileLocation(), config);
- GrowBig(m, true);
-}
-
-} // namespace
-} // namespace ngram
-} // namespace lm
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/lm_exception.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/lm_exception.cc b/ext/kenlm/lm/lm_exception.cc
deleted file mode 100644
index 58d468f..0000000
--- a/ext/kenlm/lm/lm_exception.cc
+++ /dev/null
@@ -1,23 +0,0 @@
-#include "lm/lm_exception.hh"
-
-#include <cerrno>
-#include <cstdio>
-
-namespace lm {
-
-ConfigException::ConfigException() throw() {}
-ConfigException::~ConfigException() throw() {}
-
-LoadException::LoadException() throw() {}
-LoadException::~LoadException() throw() {}
-
-FormatLoadException::FormatLoadException() throw() {}
-FormatLoadException::~FormatLoadException() throw() {}
-
-VocabLoadException::VocabLoadException() throw() {}
-VocabLoadException::~VocabLoadException() throw() {}
-
-SpecialWordMissingException::SpecialWordMissingException() throw() {}
-SpecialWordMissingException::~SpecialWordMissingException() throw() {}
-
-} // namespace lm
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/lm_exception.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/lm_exception.hh b/ext/kenlm/lm/lm_exception.hh
deleted file mode 100644
index 85a5738..0000000
--- a/ext/kenlm/lm/lm_exception.hh
+++ /dev/null
@@ -1,50 +0,0 @@
-#ifndef LM_LM_EXCEPTION_H
-#define LM_LM_EXCEPTION_H
-
-// Named to avoid conflict with util/exception.hh.
-
-#include "util/exception.hh"
-#include "util/string_piece.hh"
-
-#include <exception>
-#include <string>
-
-namespace lm {
-
-typedef enum {THROW_UP, COMPLAIN, SILENT} WarningAction;
-
-class ConfigException : public util::Exception {
- public:
- ConfigException() throw();
- ~ConfigException() throw();
-};
-
-class LoadException : public util::Exception {
- public:
- virtual ~LoadException() throw();
-
- protected:
- LoadException() throw();
-};
-
-class FormatLoadException : public LoadException {
- public:
- FormatLoadException() throw();
- ~FormatLoadException() throw();
-};
-
-class VocabLoadException : public LoadException {
- public:
- virtual ~VocabLoadException() throw();
- VocabLoadException() throw();
-};
-
-class SpecialWordMissingException : public VocabLoadException {
- public:
- explicit SpecialWordMissingException() throw();
- ~SpecialWordMissingException() throw();
-};
-
-} // namespace lm
-
-#endif // LM_LM_EXCEPTION
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/max_order.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/max_order.hh b/ext/kenlm/lm/max_order.hh
deleted file mode 100644
index 0ad1379..0000000
--- a/ext/kenlm/lm/max_order.hh
+++ /dev/null
@@ -1,13 +0,0 @@
-#ifndef LM_MAX_ORDER_H
-#define LM_MAX_ORDER_H
-/* IF YOUR BUILD SYSTEM PASSES -DKENLM_MAX_ORDER, THEN CHANGE THE BUILD SYSTEM.
- * If not, this is the default maximum order.
- * Having this limit means that State can be
- * (kMaxOrder - 1) * sizeof(float) bytes instead of
- * sizeof(float*) + (kMaxOrder - 1) * sizeof(float) + malloc overhead
- */
-#ifndef KENLM_ORDER_MESSAGE
-#define KENLM_ORDER_MESSAGE "If your build system supports changing KENLM_MAX_ORDER, change it there and recompile. In the KenLM tarball or Moses, use e.g. `bjam --max-kenlm-order=6 -a'. Otherwise, edit lm/max_order.hh."
-#endif
-
-#endif // LM_MAX_ORDER_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/model.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/model.cc b/ext/kenlm/lm/model.cc
deleted file mode 100644
index a5a16bf..0000000
--- a/ext/kenlm/lm/model.cc
+++ /dev/null
@@ -1,349 +0,0 @@
-#include "lm/model.hh"
-
-#include "lm/blank.hh"
-#include "lm/lm_exception.hh"
-#include "lm/search_hashed.hh"
-#include "lm/search_trie.hh"
-#include "lm/read_arpa.hh"
-#include "util/have.hh"
-#include "util/murmur_hash.hh"
-
-#include <algorithm>
-#include <functional>
-#include <numeric>
-#include <cmath>
-#include <limits>
-
-namespace lm {
-namespace ngram {
-namespace detail {
-
-template <class Search, class VocabularyT> const ModelType GenericModel<Search, VocabularyT>::kModelType = Search::kModelType;
-
-template <class Search, class VocabularyT> uint64_t GenericModel<Search, VocabularyT>::Size(const std::vector<uint64_t> &counts, const Config &config) {
- return VocabularyT::Size(counts[0], config) + Search::Size(counts, config);
-}
-
-template <class Search, class VocabularyT> void GenericModel<Search, VocabularyT>::SetupMemory(void *base, const std::vector<uint64_t> &counts, const Config &config) {
- size_t goal_size = util::CheckOverflow(Size(counts, config));
- uint8_t *start = static_cast<uint8_t*>(base);
- size_t allocated = VocabularyT::Size(counts[0], config);
- vocab_.SetupMemory(start, allocated, counts[0], config);
- start += allocated;
- start = search_.SetupMemory(start, counts, config);
- if (static_cast<std::size_t>(start - static_cast<uint8_t*>(base)) != goal_size) UTIL_THROW(FormatLoadException, "The data structures took " << (start - static_cast<uint8_t*>(base)) << " but Size says they should take " << goal_size);
-}
-
-namespace {
-void ComplainAboutARPA(const Config &config, ModelType model_type) {
- if (config.write_mmap || !config.messages) return;
- if (config.arpa_complain == Config::ALL) {
- *config.messages << "Loading the LM will be faster if you build a binary file." << std::endl;
- } else if (config.arpa_complain == Config::EXPENSIVE &&
- (model_type == TRIE || model_type == QUANT_TRIE || model_type == ARRAY_TRIE || model_type == QUANT_ARRAY_TRIE)) {
- *config.messages << "Building " << kModelNames[model_type] << " from ARPA is expensive. Save time by building a binary format." << std::endl;
- }
-}
-
-void CheckCounts(const std::vector<uint64_t> &counts) {
- UTIL_THROW_IF(counts.size() > KENLM_MAX_ORDER, FormatLoadException, "This model has order " << counts.size() << " but KenLM was compiled to support up to " << KENLM_MAX_ORDER << ". " << KENLM_ORDER_MESSAGE);
- if (sizeof(uint64_t) > sizeof(std::size_t)) {
- for (std::vector<uint64_t>::const_iterator i = counts.begin(); i != counts.end(); ++i) {
- UTIL_THROW_IF(*i > static_cast<uint64_t>(std::numeric_limits<size_t>::max()), util::OverflowException, "This model has " << *i << " " << (i - counts.begin() + 1) << "-grams which is too many for 32-bit machines.");
- }
- }
-}
-
-} // namespace
-
-template <class Search, class VocabularyT> GenericModel<Search, VocabularyT>::GenericModel(const char *file, const Config &init_config) : backing_(init_config) {
- util::scoped_fd fd(util::OpenReadOrThrow(file));
- if (IsBinaryFormat(fd.get())) {
- Parameters parameters;
- int fd_shallow = fd.release();
- backing_.InitializeBinary(fd_shallow, kModelType, kVersion, parameters);
- CheckCounts(parameters.counts);
-
- Config new_config(init_config);
- new_config.probing_multiplier = parameters.fixed.probing_multiplier;
- Search::UpdateConfigFromBinary(backing_, parameters.counts, VocabularyT::Size(parameters.counts[0], new_config), new_config);
- UTIL_THROW_IF(new_config.enumerate_vocab && !parameters.fixed.has_vocabulary, FormatLoadException, "The decoder requested all the vocabulary strings, but this binary file does not have them. You may need to rebuild the binary file with an updated version of build_binary.");
-
- SetupMemory(backing_.LoadBinary(Size(parameters.counts, new_config)), parameters.counts, new_config);
- vocab_.LoadedBinary(parameters.fixed.has_vocabulary, fd_shallow, new_config.enumerate_vocab, backing_.VocabStringReadingOffset());
- } else {
- ComplainAboutARPA(init_config, kModelType);
- InitializeFromARPA(fd.release(), file, init_config);
- }
-
- // g++ prints warnings unless these are fully initialized.
- State begin_sentence = State();
- begin_sentence.length = 1;
- begin_sentence.words[0] = vocab_.BeginSentence();
- typename Search::Node ignored_node;
- bool ignored_independent_left;
- uint64_t ignored_extend_left;
- begin_sentence.backoff[0] = search_.LookupUnigram(begin_sentence.words[0], ignored_node, ignored_independent_left, ignored_extend_left).Backoff();
- State null_context = State();
- null_context.length = 0;
- P::Init(begin_sentence, null_context, vocab_, search_.Order());
-}
-
-template <class Search, class VocabularyT> void GenericModel<Search, VocabularyT>::InitializeFromARPA(int fd, const char *file, const Config &config) {
- // Backing file is the ARPA.
- util::FilePiece f(fd, file, config.ProgressMessages());
- try {
- std::vector<uint64_t> counts;
- // File counts do not include pruned trigrams that extend to quadgrams etc. These will be fixed by search_.
- ReadARPACounts(f, counts);
- CheckCounts(counts);
- if (counts.size() < 2) UTIL_THROW(FormatLoadException, "This ngram implementation assumes at least a bigram model.");
- if (config.probing_multiplier <= 1.0) UTIL_THROW(ConfigException, "probing multiplier must be > 1.0");
-
- std::size_t vocab_size = util::CheckOverflow(VocabularyT::Size(counts[0], config));
- // Setup the binary file for writing the vocab lookup table. The search_ is responsible for growing the binary file to its needs.
- vocab_.SetupMemory(backing_.SetupJustVocab(vocab_size, counts.size()), vocab_size, counts[0], config);
-
- if (config.write_mmap && config.include_vocab) {
- WriteWordsWrapper wrap(config.enumerate_vocab);
- vocab_.ConfigureEnumerate(&wrap, counts[0]);
- search_.InitializeFromARPA(file, f, counts, config, vocab_, backing_);
- void *vocab_rebase, *search_rebase;
- backing_.WriteVocabWords(wrap.Buffer(), vocab_rebase, search_rebase);
- // Due to writing at the end of file, mmap may have relocated data. So remap.
- vocab_.Relocate(vocab_rebase);
- search_.SetupMemory(reinterpret_cast<uint8_t*>(search_rebase), counts, config);
- } else {
- vocab_.ConfigureEnumerate(config.enumerate_vocab, counts[0]);
- search_.InitializeFromARPA(file, f, counts, config, vocab_, backing_);
- }
-
- if (!vocab_.SawUnk()) {
- assert(config.unknown_missing != THROW_UP);
- // Default probabilities for unknown.
- search_.UnknownUnigram().backoff = 0.0;
- search_.UnknownUnigram().prob = config.unknown_missing_logprob;
- }
- backing_.FinishFile(config, kModelType, kVersion, counts);
- } catch (util::Exception &e) {
- e << " Byte: " << f.Offset();
- throw;
- }
-}
-
-template <class Search, class VocabularyT> FullScoreReturn GenericModel<Search, VocabularyT>::FullScore(const State &in_state, const WordIndex new_word, State &out_state) const {
- FullScoreReturn ret = ScoreExceptBackoff(in_state.words, in_state.words + in_state.length, new_word, out_state);
- for (const float *i = in_state.backoff + ret.ngram_length - 1; i < in_state.backoff + in_state.length; ++i) {
- ret.prob += *i;
- }
- return ret;
-}
-
-template <class Search, class VocabularyT> FullScoreReturn GenericModel<Search, VocabularyT>::FullScoreForgotState(const WordIndex *context_rbegin, const WordIndex *context_rend, const WordIndex new_word, State &out_state) const {
- context_rend = std::min(context_rend, context_rbegin + P::Order() - 1);
- FullScoreReturn ret = ScoreExceptBackoff(context_rbegin, context_rend, new_word, out_state);
-
- // Add the backoff weights for n-grams of order start to (context_rend - context_rbegin).
- unsigned char start = ret.ngram_length;
- if (context_rend - context_rbegin < static_cast<std::ptrdiff_t>(start)) return ret;
-
- bool independent_left;
- uint64_t extend_left;
- typename Search::Node node;
- if (start <= 1) {
- ret.prob += search_.LookupUnigram(*context_rbegin, node, independent_left, extend_left).Backoff();
- start = 2;
- } else if (!search_.FastMakeNode(context_rbegin, context_rbegin + start - 1, node)) {
- return ret;
- }
- // i is the order of the backoff we're looking for.
- unsigned char order_minus_2 = start - 2;
- for (const WordIndex *i = context_rbegin + start - 1; i < context_rend; ++i, ++order_minus_2) {
- typename Search::MiddlePointer p(search_.LookupMiddle(order_minus_2, *i, node, independent_left, extend_left));
- if (!p.Found()) break;
- ret.prob += p.Backoff();
- }
- return ret;
-}
-
-template <class Search, class VocabularyT> void GenericModel<Search, VocabularyT>::GetState(const WordIndex *context_rbegin, const WordIndex *context_rend, State &out_state) const {
- // Generate a state from context.
- context_rend = std::min(context_rend, context_rbegin + P::Order() - 1);
- if (context_rend == context_rbegin) {
- out_state.length = 0;
- return;
- }
- typename Search::Node node;
- bool independent_left;
- uint64_t extend_left;
- out_state.backoff[0] = search_.LookupUnigram(*context_rbegin, node, independent_left, extend_left).Backoff();
- out_state.length = HasExtension(out_state.backoff[0]) ? 1 : 0;
- float *backoff_out = out_state.backoff + 1;
- unsigned char order_minus_2 = 0;
- for (const WordIndex *i = context_rbegin + 1; i < context_rend; ++i, ++backoff_out, ++order_minus_2) {
- typename Search::MiddlePointer p(search_.LookupMiddle(order_minus_2, *i, node, independent_left, extend_left));
- if (!p.Found()) {
- std::copy(context_rbegin, context_rbegin + out_state.length, out_state.words);
- return;
- }
- *backoff_out = p.Backoff();
- if (HasExtension(*backoff_out)) out_state.length = i - context_rbegin + 1;
- }
- std::copy(context_rbegin, context_rbegin + out_state.length, out_state.words);
-}
-
-template <class Search, class VocabularyT> FullScoreReturn GenericModel<Search, VocabularyT>::ExtendLeft(
- const WordIndex *add_rbegin, const WordIndex *add_rend,
- const float *backoff_in,
- uint64_t extend_pointer,
- unsigned char extend_length,
- float *backoff_out,
- unsigned char &next_use) const {
- FullScoreReturn ret;
- typename Search::Node node;
- if (extend_length == 1) {
- typename Search::UnigramPointer ptr(search_.LookupUnigram(static_cast<WordIndex>(extend_pointer), node, ret.independent_left, ret.extend_left));
- ret.rest = ptr.Rest();
- ret.prob = ptr.Prob();
- assert(!ret.independent_left);
- } else {
- typename Search::MiddlePointer ptr(search_.Unpack(extend_pointer, extend_length, node));
- ret.rest = ptr.Rest();
- ret.prob = ptr.Prob();
- ret.extend_left = extend_pointer;
- // If this function is called, then it does depend on left words.
- ret.independent_left = false;
- }
- float subtract_me = ret.rest;
- ret.ngram_length = extend_length;
- next_use = extend_length;
- ResumeScore(add_rbegin, add_rend, extend_length - 1, node, backoff_out, next_use, ret);
- next_use -= extend_length;
- // Charge backoffs.
- for (const float *b = backoff_in + ret.ngram_length - extend_length; b < backoff_in + (add_rend - add_rbegin); ++b) ret.prob += *b;
- ret.prob -= subtract_me;
- ret.rest -= subtract_me;
- return ret;
-}
-
-namespace {
-// Do a paraonoid copy of history, assuming new_word has already been copied
-// (hence the -1). out_state.length could be zero so I avoided using
-// std::copy.
-void CopyRemainingHistory(const WordIndex *from, State &out_state) {
- WordIndex *out = out_state.words + 1;
- const WordIndex *in_end = from + static_cast<ptrdiff_t>(out_state.length) - 1;
- for (const WordIndex *in = from; in < in_end; ++in, ++out) *out = *in;
-}
-} // namespace
-
-/* Ugly optimized function. Produce a score excluding backoff.
- * The search goes in increasing order of ngram length.
- * Context goes backward, so context_begin is the word immediately preceeding
- * new_word.
- */
-template <class Search, class VocabularyT> FullScoreReturn GenericModel<Search, VocabularyT>::ScoreExceptBackoff(
- const WordIndex *const context_rbegin,
- const WordIndex *const context_rend,
- const WordIndex new_word,
- State &out_state) const {
- assert(new_word < vocab_.Bound());
- FullScoreReturn ret;
- // ret.ngram_length contains the last known non-blank ngram length.
- ret.ngram_length = 1;
-
- typename Search::Node node;
- typename Search::UnigramPointer uni(search_.LookupUnigram(new_word, node, ret.independent_left, ret.extend_left));
- out_state.backoff[0] = uni.Backoff();
- ret.prob = uni.Prob();
- ret.rest = uni.Rest();
-
- // This is the length of the context that should be used for continuation to the right.
- out_state.length = HasExtension(out_state.backoff[0]) ? 1 : 0;
- // We'll write the word anyway since it will probably be used and does no harm being there.
- out_state.words[0] = new_word;
- if (context_rbegin == context_rend) return ret;
-
- ResumeScore(context_rbegin, context_rend, 0, node, out_state.backoff + 1, out_state.length, ret);
- CopyRemainingHistory(context_rbegin, out_state);
- return ret;
-}
-
-template <class Search, class VocabularyT> void GenericModel<Search, VocabularyT>::ResumeScore(const WordIndex *hist_iter, const WordIndex *const context_rend, unsigned char order_minus_2, typename Search::Node &node, float *backoff_out, unsigned char &next_use, FullScoreReturn &ret) const {
- for (; ; ++order_minus_2, ++hist_iter, ++backoff_out) {
- if (hist_iter == context_rend) return;
- if (ret.independent_left) return;
- if (order_minus_2 == P::Order() - 2) break;
-
- typename Search::MiddlePointer pointer(search_.LookupMiddle(order_minus_2, *hist_iter, node, ret.independent_left, ret.extend_left));
- if (!pointer.Found()) return;
- *backoff_out = pointer.Backoff();
- ret.prob = pointer.Prob();
- ret.rest = pointer.Rest();
- ret.ngram_length = order_minus_2 + 2;
- if (HasExtension(*backoff_out)) {
- next_use = ret.ngram_length;
- }
- }
- ret.independent_left = true;
- typename Search::LongestPointer longest(search_.LookupLongest(*hist_iter, node));
- if (longest.Found()) {
- ret.prob = longest.Prob();
- ret.rest = ret.prob;
- // There is no blank in longest_.
- ret.ngram_length = P::Order();
- }
-}
-
-template <class Search, class VocabularyT> float GenericModel<Search, VocabularyT>::InternalUnRest(const uint64_t *pointers_begin, const uint64_t *pointers_end, unsigned char first_length) const {
- float ret;
- typename Search::Node node;
- if (first_length == 1) {
- if (pointers_begin >= pointers_end) return 0.0;
- bool independent_left;
- uint64_t extend_left;
- typename Search::UnigramPointer ptr(search_.LookupUnigram(static_cast<WordIndex>(*pointers_begin), node, independent_left, extend_left));
- ret = ptr.Prob() - ptr.Rest();
- ++first_length;
- ++pointers_begin;
- } else {
- ret = 0.0;
- }
- for (const uint64_t *i = pointers_begin; i < pointers_end; ++i, ++first_length) {
- typename Search::MiddlePointer ptr(search_.Unpack(*i, first_length, node));
- ret += ptr.Prob() - ptr.Rest();
- }
- return ret;
-}
-
-template class GenericModel<HashedSearch<BackoffValue>, ProbingVocabulary>;
-template class GenericModel<HashedSearch<RestValue>, ProbingVocabulary>;
-template class GenericModel<trie::TrieSearch<DontQuantize, trie::DontBhiksha>, SortedVocabulary>;
-template class GenericModel<trie::TrieSearch<DontQuantize, trie::ArrayBhiksha>, SortedVocabulary>;
-template class GenericModel<trie::TrieSearch<SeparatelyQuantize, trie::DontBhiksha>, SortedVocabulary>;
-template class GenericModel<trie::TrieSearch<SeparatelyQuantize, trie::ArrayBhiksha>, SortedVocabulary>;
-
-} // namespace detail
-
-base::Model *LoadVirtual(const char *file_name, const Config &config, ModelType model_type) {
- RecognizeBinary(file_name, model_type);
- switch (model_type) {
- case PROBING:
- return new ProbingModel(file_name, config);
- case REST_PROBING:
- return new RestProbingModel(file_name, config);
- case TRIE:
- return new TrieModel(file_name, config);
- case QUANT_TRIE:
- return new QuantTrieModel(file_name, config);
- case ARRAY_TRIE:
- return new ArrayTrieModel(file_name, config);
- case QUANT_ARRAY_TRIE:
- return new QuantArrayTrieModel(file_name, config);
- default:
- UTIL_THROW(FormatLoadException, "Confused by model type " << model_type);
- }
-}
-
-} // namespace ngram
-} // namespace lm
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/model.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/model.hh b/ext/kenlm/lm/model.hh
deleted file mode 100644
index b2bbe39..0000000
--- a/ext/kenlm/lm/model.hh
+++ /dev/null
@@ -1,155 +0,0 @@
-#ifndef LM_MODEL_H
-#define LM_MODEL_H
-
-#include "lm/bhiksha.hh"
-#include "lm/binary_format.hh"
-#include "lm/config.hh"
-#include "lm/facade.hh"
-#include "lm/quantize.hh"
-#include "lm/search_hashed.hh"
-#include "lm/search_trie.hh"
-#include "lm/state.hh"
-#include "lm/value.hh"
-#include "lm/vocab.hh"
-#include "lm/weights.hh"
-
-#include "util/murmur_hash.hh"
-
-#include <algorithm>
-#include <vector>
-#include <cstring>
-
-namespace util { class FilePiece; }
-
-namespace lm {
-namespace ngram {
-namespace detail {
-
-// Should return the same results as SRI.
-// ModelFacade typedefs Vocabulary so we use VocabularyT to avoid naming conflicts.
-template <class Search, class VocabularyT> class GenericModel : public base::ModelFacade<GenericModel<Search, VocabularyT>, State, VocabularyT> {
- private:
- typedef base::ModelFacade<GenericModel<Search, VocabularyT>, State, VocabularyT> P;
- public:
- // This is the model type returned by RecognizeBinary.
- static const ModelType kModelType;
-
- static const unsigned int kVersion = Search::kVersion;
-
- /* Get the size of memory that will be mapped given ngram counts. This
- * does not include small non-mapped control structures, such as this class
- * itself.
- */
- static uint64_t Size(const std::vector<uint64_t> &counts, const Config &config = Config());
-
- /* Load the model from a file. It may be an ARPA or binary file. Binary
- * files must have the format expected by this class or you'll get an
- * exception. So TrieModel can only load ARPA or binary created by
- * TrieModel. To classify binary files, call RecognizeBinary in
- * lm/binary_format.hh.
- */
- explicit GenericModel(const char *file, const Config &config = Config());
-
- /* Score p(new_word | in_state) and incorporate new_word into out_state.
- * Note that in_state and out_state must be different references:
- * &in_state != &out_state.
- */
- FullScoreReturn FullScore(const State &in_state, const WordIndex new_word, State &out_state) const;
-
- /* Slower call without in_state. Try to remember state, but sometimes it
- * would cost too much memory or your decoder isn't setup properly.
- * To use this function, make an array of WordIndex containing the context
- * vocabulary ids in reverse order. Then, pass the bounds of the array:
- * [context_rbegin, context_rend). The new_word is not part of the context
- * array unless you intend to repeat words.
- */
- FullScoreReturn FullScoreForgotState(const WordIndex *context_rbegin, const WordIndex *context_rend, const WordIndex new_word, State &out_state) const;
-
- /* Get the state for a context. Don't use this if you can avoid it. Use
- * BeginSentenceState or NullContextState and extend from those. If
- * you're only going to use this state to call FullScore once, use
- * FullScoreForgotState.
- * To use this function, make an array of WordIndex containing the context
- * vocabulary ids in reverse order. Then, pass the bounds of the array:
- * [context_rbegin, context_rend).
- */
- void GetState(const WordIndex *context_rbegin, const WordIndex *context_rend, State &out_state) const;
-
- /* More efficient version of FullScore where a partial n-gram has already
- * been scored.
- * NOTE: THE RETURNED .rest AND .prob ARE RELATIVE TO THE .rest RETURNED BEFORE.
- */
- FullScoreReturn ExtendLeft(
- // Additional context in reverse order. This will update add_rend to
- const WordIndex *add_rbegin, const WordIndex *add_rend,
- // Backoff weights to use.
- const float *backoff_in,
- // extend_left returned by a previous query.
- uint64_t extend_pointer,
- // Length of n-gram that the pointer corresponds to.
- unsigned char extend_length,
- // Where to write additional backoffs for [extend_length + 1, min(Order() - 1, return.ngram_length)]
- float *backoff_out,
- // Amount of additional content that should be considered by the next call.
- unsigned char &next_use) const;
-
- /* Return probabilities minus rest costs for an array of pointers. The
- * first length should be the length of the n-gram to which pointers_begin
- * points.
- */
- float UnRest(const uint64_t *pointers_begin, const uint64_t *pointers_end, unsigned char first_length) const {
- // Compiler should optimize this if away.
- return Search::kDifferentRest ? InternalUnRest(pointers_begin, pointers_end, first_length) : 0.0;
- }
-
- private:
- FullScoreReturn ScoreExceptBackoff(const WordIndex *const context_rbegin, const WordIndex *const context_rend, const WordIndex new_word, State &out_state) const;
-
- // Score bigrams and above. Do not include backoff.
- void ResumeScore(const WordIndex *context_rbegin, const WordIndex *const context_rend, unsigned char starting_order_minus_2, typename Search::Node &node, float *backoff_out, unsigned char &next_use, FullScoreReturn &ret) const;
-
- // Appears after Size in the cc file.
- void SetupMemory(void *start, const std::vector<uint64_t> &counts, const Config &config);
-
- void InitializeFromARPA(int fd, const char *file, const Config &config);
-
- float InternalUnRest(const uint64_t *pointers_begin, const uint64_t *pointers_end, unsigned char first_length) const;
-
- BinaryFormat backing_;
-
- VocabularyT vocab_;
-
- Search search_;
-};
-
-} // namespace detail
-
-// Instead of typedef, inherit. This allows the Model etc to be forward declared.
-// Oh the joys of C and C++.
-#define LM_COMMA() ,
-#define LM_NAME_MODEL(name, from)\
-class name : public from {\
- public:\
- name(const char *file, const Config &config = Config()) : from(file, config) {}\
-};
-
-LM_NAME_MODEL(ProbingModel, detail::GenericModel<detail::HashedSearch<BackoffValue> LM_COMMA() ProbingVocabulary>);
-LM_NAME_MODEL(RestProbingModel, detail::GenericModel<detail::HashedSearch<RestValue> LM_COMMA() ProbingVocabulary>);
-LM_NAME_MODEL(TrieModel, detail::GenericModel<trie::TrieSearch<DontQuantize LM_COMMA() trie::DontBhiksha> LM_COMMA() SortedVocabulary>);
-LM_NAME_MODEL(ArrayTrieModel, detail::GenericModel<trie::TrieSearch<DontQuantize LM_COMMA() trie::ArrayBhiksha> LM_COMMA() SortedVocabulary>);
-LM_NAME_MODEL(QuantTrieModel, detail::GenericModel<trie::TrieSearch<SeparatelyQuantize LM_COMMA() trie::DontBhiksha> LM_COMMA() SortedVocabulary>);
-LM_NAME_MODEL(QuantArrayTrieModel, detail::GenericModel<trie::TrieSearch<SeparatelyQuantize LM_COMMA() trie::ArrayBhiksha> LM_COMMA() SortedVocabulary>);
-
-// Default implementation. No real reason for it to be the default.
-typedef ::lm::ngram::ProbingVocabulary Vocabulary;
-typedef ProbingModel Model;
-
-/* Autorecognize the file type, load, and return the virtual base class. Don't
- * use the virtual base class if you can avoid it. Instead, use the above
- * classes as template arguments to your own virtual feature function.*/
-base::Model *LoadVirtual(const char *file_name, const Config &config = Config(), ModelType if_arpa = PROBING);
-
-} // namespace ngram
-} // namespace lm
-
-#endif // LM_MODEL_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/model_test.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/model_test.cc b/ext/kenlm/lm/model_test.cc
deleted file mode 100644
index d408d6f..0000000
--- a/ext/kenlm/lm/model_test.cc
+++ /dev/null
@@ -1,448 +0,0 @@
-#include "lm/model.hh"
-
-#include <cstdlib>
-#include <cstring>
-
-#define BOOST_TEST_MODULE ModelTest
-#include <boost/test/unit_test.hpp>
-#include <boost/test/floating_point_comparison.hpp>
-
-// Apparently some Boost versions use templates and are pretty strict about types matching.
-#define SLOPPY_CHECK_CLOSE(ref, value, tol) BOOST_CHECK_CLOSE(static_cast<double>(ref), static_cast<double>(value), static_cast<double>(tol));
-
-namespace lm {
-namespace ngram {
-
-std::ostream &operator<<(std::ostream &o, const State &state) {
- o << "State length " << static_cast<unsigned int>(state.length) << ':';
- for (const WordIndex *i = state.words; i < state.words + state.length; ++i) {
- o << ' ' << *i;
- }
- return o;
-}
-
-namespace {
-
-// Stupid bjam reverses the command line arguments randomly.
-const char *TestLocation() {
- if (boost::unit_test::framework::master_test_suite().argc < 3) {
- return "test.arpa";
- }
- char **argv = boost::unit_test::framework::master_test_suite().argv;
- return argv[strstr(argv[1], "nounk") ? 2 : 1];
-}
-const char *TestNoUnkLocation() {
- if (boost::unit_test::framework::master_test_suite().argc < 3) {
- return "test_nounk.arpa";
- }
- char **argv = boost::unit_test::framework::master_test_suite().argv;
- return argv[strstr(argv[1], "nounk") ? 1 : 2];
-}
-
-template <class Model> State GetState(const Model &model, const char *word, const State &in) {
- WordIndex context[in.length + 1];
- context[0] = model.GetVocabulary().Index(word);
- std::copy(in.words, in.words + in.length, context + 1);
- State ret;
- model.GetState(context, context + in.length + 1, ret);
- return ret;
-}
-
-#define StartTest(word, ngram, score, indep_left) \
- ret = model.FullScore( \
- state, \
- model.GetVocabulary().Index(word), \
- out);\
- SLOPPY_CHECK_CLOSE(score, ret.prob, 0.001); \
- BOOST_CHECK_EQUAL(static_cast<unsigned int>(ngram), ret.ngram_length); \
- BOOST_CHECK_GE(std::min<unsigned char>(ngram, 5 - 1), out.length); \
- BOOST_CHECK_EQUAL(indep_left, ret.independent_left); \
- BOOST_CHECK_EQUAL(out, GetState(model, word, state));
-
-#define AppendTest(word, ngram, score, indep_left) \
- StartTest(word, ngram, score, indep_left) \
- state = out;
-
-template <class M> void Starters(const M &model) {
- FullScoreReturn ret;
- Model::State state(model.BeginSentenceState());
- Model::State out;
-
- StartTest("looking", 2, -0.4846522, true);
-
- // , probability plus <s> backoff
- StartTest(",", 1, -1.383514 + -0.4149733, true);
- // <unk> probability plus <s> backoff
- StartTest("this_is_not_found", 1, -1.995635 + -0.4149733, true);
-}
-
-template <class M> void Continuation(const M &model) {
- FullScoreReturn ret;
- Model::State state(model.BeginSentenceState());
- Model::State out;
-
- AppendTest("looking", 2, -0.484652, true);
- AppendTest("on", 3, -0.348837, true);
- AppendTest("a", 4, -0.0155266, true);
- AppendTest("little", 5, -0.00306122, true);
- State preserve = state;
- AppendTest("the", 1, -4.04005, true);
- AppendTest("biarritz", 1, -1.9889, true);
- AppendTest("not_found", 1, -2.29666, true);
- AppendTest("more", 1, -1.20632 - 20.0, true);
- AppendTest(".", 2, -0.51363, true);
- AppendTest("</s>", 3, -0.0191651, true);
- BOOST_CHECK_EQUAL(0, state.length);
-
- state = preserve;
- AppendTest("more", 5, -0.00181395, true);
- BOOST_CHECK_EQUAL(4, state.length);
- AppendTest("loin", 5, -0.0432557, true);
- BOOST_CHECK_EQUAL(1, state.length);
-}
-
-template <class M> void Blanks(const M &model) {
- FullScoreReturn ret;
- State state(model.NullContextState());
- State out;
- AppendTest("also", 1, -1.687872, false);
- AppendTest("would", 2, -2, true);
- AppendTest("consider", 3, -3, true);
- State preserve = state;
- AppendTest("higher", 4, -4, true);
- AppendTest("looking", 5, -5, true);
- BOOST_CHECK_EQUAL(1, state.length);
-
- state = preserve;
- // also would consider not_found
- AppendTest("not_found", 1, -1.995635 - 7.0 - 0.30103, true);
-
- state = model.NullContextState();
- // higher looking is a blank.
- AppendTest("higher", 1, -1.509559, false);
- AppendTest("looking", 2, -1.285941 - 0.30103, false);
-
- State higher_looking = state;
-
- BOOST_CHECK_EQUAL(1, state.length);
- AppendTest("not_found", 1, -1.995635 - 0.4771212, true);
-
- state = higher_looking;
- // higher looking consider
- AppendTest("consider", 1, -1.687872 - 0.4771212, true);
-
- state = model.NullContextState();
- AppendTest("would", 1, -1.687872, false);
- BOOST_CHECK_EQUAL(1, state.length);
- AppendTest("consider", 2, -1.687872 -0.30103, false);
- BOOST_CHECK_EQUAL(2, state.length);
- AppendTest("higher", 3, -1.509559 - 0.30103, false);
- BOOST_CHECK_EQUAL(3, state.length);
- AppendTest("looking", 4, -1.285941 - 0.30103, false);
-}
-
-template <class M> void Unknowns(const M &model) {
- FullScoreReturn ret;
- State state(model.NullContextState());
- State out;
-
- AppendTest("not_found", 1, -1.995635, false);
- State preserve = state;
- AppendTest("not_found2", 2, -15.0, true);
- AppendTest("not_found3", 2, -15.0 - 2.0, true);
-
- state = preserve;
- AppendTest("however", 2, -4, true);
- AppendTest("not_found3", 3, -6, true);
-}
-
-template <class M> void MinimalState(const M &model) {
- FullScoreReturn ret;
- State state(model.NullContextState());
- State out;
-
- AppendTest("baz", 1, -6.535897, true);
- BOOST_CHECK_EQUAL(0, state.length);
- state = model.NullContextState();
- AppendTest("foo", 1, -3.141592, true);
- BOOST_CHECK_EQUAL(1, state.length);
- AppendTest("bar", 2, -6.0, true);
- // Has to include the backoff weight.
- BOOST_CHECK_EQUAL(1, state.length);
- AppendTest("bar", 1, -2.718281 + 3.0, true);
- BOOST_CHECK_EQUAL(1, state.length);
-
- state = model.NullContextState();
- AppendTest("to", 1, -1.687872, false);
- AppendTest("look", 2, -0.2922095, true);
- BOOST_CHECK_EQUAL(2, state.length);
- AppendTest("a", 3, -7, true);
-}
-
-template <class M> void ExtendLeftTest(const M &model) {
- State right;
- FullScoreReturn little(model.FullScore(model.NullContextState(), model.GetVocabulary().Index("little"), right));
- const float kLittleProb = -1.285941;
- SLOPPY_CHECK_CLOSE(kLittleProb, little.prob, 0.001);
- unsigned char next_use;
- float backoff_out[4];
-
- FullScoreReturn extend_none(model.ExtendLeft(NULL, NULL, NULL, little.extend_left, 1, NULL, next_use));
- BOOST_CHECK_EQUAL(0, next_use);
- BOOST_CHECK_EQUAL(little.extend_left, extend_none.extend_left);
- SLOPPY_CHECK_CLOSE(little.prob - little.rest, extend_none.prob, 0.001);
- BOOST_CHECK_EQUAL(1, extend_none.ngram_length);
-
- const WordIndex a = model.GetVocabulary().Index("a");
- float backoff_in = 3.14;
- // a little
- FullScoreReturn extend_a(model.ExtendLeft(&a, &a + 1, &backoff_in, little.extend_left, 1, backoff_out, next_use));
- BOOST_CHECK_EQUAL(1, next_use);
- SLOPPY_CHECK_CLOSE(-0.69897, backoff_out[0], 0.001);
- SLOPPY_CHECK_CLOSE(-0.09132547 - little.rest, extend_a.prob, 0.001);
- BOOST_CHECK_EQUAL(2, extend_a.ngram_length);
- BOOST_CHECK(!extend_a.independent_left);
-
- const WordIndex on = model.GetVocabulary().Index("on");
- FullScoreReturn extend_on(model.ExtendLeft(&on, &on + 1, &backoff_in, extend_a.extend_left, 2, backoff_out, next_use));
- BOOST_CHECK_EQUAL(1, next_use);
- SLOPPY_CHECK_CLOSE(-0.4771212, backoff_out[0], 0.001);
- SLOPPY_CHECK_CLOSE(-0.0283603 - (extend_a.rest + little.rest), extend_on.prob, 0.001);
- BOOST_CHECK_EQUAL(3, extend_on.ngram_length);
- BOOST_CHECK(!extend_on.independent_left);
-
- const WordIndex both[2] = {a, on};
- float backoff_in_arr[4];
- FullScoreReturn extend_both(model.ExtendLeft(both, both + 2, backoff_in_arr, little.extend_left, 1, backoff_out, next_use));
- BOOST_CHECK_EQUAL(2, next_use);
- SLOPPY_CHECK_CLOSE(-0.69897, backoff_out[0], 0.001);
- SLOPPY_CHECK_CLOSE(-0.4771212, backoff_out[1], 0.001);
- SLOPPY_CHECK_CLOSE(-0.0283603 - little.rest, extend_both.prob, 0.001);
- BOOST_CHECK_EQUAL(3, extend_both.ngram_length);
- BOOST_CHECK(!extend_both.independent_left);
- BOOST_CHECK_EQUAL(extend_on.extend_left, extend_both.extend_left);
-}
-
-#define StatelessTest(word, provide, ngram, score) \
- ret = model.FullScoreForgotState(indices + num_words - word, indices + num_words - word + provide, indices[num_words - word - 1], state); \
- SLOPPY_CHECK_CLOSE(score, ret.prob, 0.001); \
- BOOST_CHECK_EQUAL(static_cast<unsigned int>(ngram), ret.ngram_length); \
- model.GetState(indices + num_words - word, indices + num_words - word + provide, before); \
- ret = model.FullScore(before, indices[num_words - word - 1], out); \
- BOOST_CHECK(state == out); \
- SLOPPY_CHECK_CLOSE(score, ret.prob, 0.001); \
- BOOST_CHECK_EQUAL(static_cast<unsigned int>(ngram), ret.ngram_length);
-
-template <class M> void Stateless(const M &model) {
- const char *words[] = {"<s>", "looking", "on", "a", "little", "the", "biarritz", "not_found", "more", ".", "</s>"};
- const size_t num_words = sizeof(words) / sizeof(const char*);
- // Silience "array subscript is above array bounds" when extracting end pointer.
- WordIndex indices[num_words + 1];
- for (unsigned int i = 0; i < num_words; ++i) {
- indices[num_words - 1 - i] = model.GetVocabulary().Index(words[i]);
- }
- FullScoreReturn ret;
- State state, out, before;
-
- ret = model.FullScoreForgotState(indices + num_words - 1, indices + num_words, indices[num_words - 2], state);
- SLOPPY_CHECK_CLOSE(-0.484652, ret.prob, 0.001);
- StatelessTest(1, 1, 2, -0.484652);
-
- // looking
- StatelessTest(1, 2, 2, -0.484652);
- // on
- AppendTest("on", 3, -0.348837, true);
- StatelessTest(2, 3, 3, -0.348837);
- StatelessTest(2, 2, 3, -0.348837);
- StatelessTest(2, 1, 2, -0.4638903);
- // a
- StatelessTest(3, 4, 4, -0.0155266);
- // little
- AppendTest("little", 5, -0.00306122, true);
- StatelessTest(4, 5, 5, -0.00306122);
- // the
- AppendTest("the", 1, -4.04005, true);
- StatelessTest(5, 5, 1, -4.04005);
- // No context of the.
- StatelessTest(5, 0, 1, -1.687872);
- // biarritz
- StatelessTest(6, 1, 1, -1.9889);
- // not found
- StatelessTest(7, 1, 1, -2.29666);
- StatelessTest(7, 0, 1, -1.995635);
-
- WordIndex unk[1];
- unk[0] = 0;
- model.GetState(unk, unk + 1, state);
- BOOST_CHECK_EQUAL(1, state.length);
- BOOST_CHECK_EQUAL(static_cast<WordIndex>(0), state.words[0]);
-}
-
-template <class M> void NoUnkCheck(const M &model) {
- WordIndex unk_index = 0;
- State state;
-
- FullScoreReturn ret = model.FullScoreForgotState(&unk_index, &unk_index + 1, unk_index, state);
- SLOPPY_CHECK_CLOSE(-100.0, ret.prob, 0.001);
-}
-
-template <class M> void Everything(const M &m) {
- Starters(m);
- Continuation(m);
- Blanks(m);
- Unknowns(m);
- MinimalState(m);
- ExtendLeftTest(m);
- Stateless(m);
-}
-
-class ExpectEnumerateVocab : public EnumerateVocab {
- public:
- ExpectEnumerateVocab() {}
-
- void Add(WordIndex index, const StringPiece &str) {
- BOOST_CHECK_EQUAL(seen.size(), index);
- seen.push_back(std::string(str.data(), str.length()));
- }
-
- void Check(const base::Vocabulary &vocab) {
- BOOST_CHECK_EQUAL(37ULL, seen.size());
- BOOST_REQUIRE(!seen.empty());
- BOOST_CHECK_EQUAL("<unk>", seen[0]);
- for (WordIndex i = 0; i < seen.size(); ++i) {
- BOOST_CHECK_EQUAL(i, vocab.Index(seen[i]));
- }
- }
-
- void Clear() {
- seen.clear();
- }
-
- std::vector<std::string> seen;
-};
-
-template <class ModelT> void LoadingTest() {
- Config config;
- config.arpa_complain = Config::NONE;
- config.messages = NULL;
- config.probing_multiplier = 2.0;
- {
- ExpectEnumerateVocab enumerate;
- config.enumerate_vocab = &enumerate;
- ModelT m(TestLocation(), config);
- enumerate.Check(m.GetVocabulary());
- BOOST_CHECK_EQUAL((WordIndex)37, m.GetVocabulary().Bound());
- Everything(m);
- }
- {
- ExpectEnumerateVocab enumerate;
- config.enumerate_vocab = &enumerate;
- ModelT m(TestNoUnkLocation(), config);
- enumerate.Check(m.GetVocabulary());
- BOOST_CHECK_EQUAL((WordIndex)37, m.GetVocabulary().Bound());
- NoUnkCheck(m);
- }
-}
-
-BOOST_AUTO_TEST_CASE(probing) {
- LoadingTest<Model>();
-}
-BOOST_AUTO_TEST_CASE(trie) {
- LoadingTest<TrieModel>();
-}
-BOOST_AUTO_TEST_CASE(quant_trie) {
- LoadingTest<QuantTrieModel>();
-}
-BOOST_AUTO_TEST_CASE(bhiksha_trie) {
- LoadingTest<ArrayTrieModel>();
-}
-BOOST_AUTO_TEST_CASE(quant_bhiksha_trie) {
- LoadingTest<QuantArrayTrieModel>();
-}
-
-template <class ModelT> void BinaryTest(Config::WriteMethod write_method) {
- Config config;
- config.write_mmap = "test.binary";
- config.messages = NULL;
- config.write_method = write_method;
- ExpectEnumerateVocab enumerate;
- config.enumerate_vocab = &enumerate;
-
- {
- ModelT copy_model(TestLocation(), config);
- enumerate.Check(copy_model.GetVocabulary());
- enumerate.Clear();
- Everything(copy_model);
- }
-
- config.write_mmap = NULL;
-
- ModelType type;
- BOOST_REQUIRE(RecognizeBinary("test.binary", type));
- BOOST_CHECK_EQUAL(ModelT::kModelType, type);
-
- {
- ModelT binary("test.binary", config);
- enumerate.Check(binary.GetVocabulary());
- Everything(binary);
- }
- unlink("test.binary");
-
- // Now test without <unk>.
- config.write_mmap = "test_nounk.binary";
- config.messages = NULL;
- enumerate.Clear();
- {
- ModelT copy_model(TestNoUnkLocation(), config);
- enumerate.Check(copy_model.GetVocabulary());
- enumerate.Clear();
- NoUnkCheck(copy_model);
- }
- config.write_mmap = NULL;
- {
- ModelT binary(TestNoUnkLocation(), config);
- enumerate.Check(binary.GetVocabulary());
- NoUnkCheck(binary);
- }
- unlink("test_nounk.binary");
-}
-
-template <class ModelT> void BinaryTest() {
- BinaryTest<ModelT>(Config::WRITE_MMAP);
- BinaryTest<ModelT>(Config::WRITE_AFTER);
-}
-
-BOOST_AUTO_TEST_CASE(write_and_read_probing) {
- BinaryTest<ProbingModel>();
-}
-BOOST_AUTO_TEST_CASE(write_and_read_rest_probing) {
- BinaryTest<RestProbingModel>();
-}
-BOOST_AUTO_TEST_CASE(write_and_read_trie) {
- BinaryTest<TrieModel>();
-}
-BOOST_AUTO_TEST_CASE(write_and_read_quant_trie) {
- BinaryTest<QuantTrieModel>();
-}
-BOOST_AUTO_TEST_CASE(write_and_read_array_trie) {
- BinaryTest<ArrayTrieModel>();
-}
-BOOST_AUTO_TEST_CASE(write_and_read_quant_array_trie) {
- BinaryTest<QuantArrayTrieModel>();
-}
-
-BOOST_AUTO_TEST_CASE(rest_max) {
- Config config;
- config.arpa_complain = Config::NONE;
- config.messages = NULL;
-
- RestProbingModel model(TestLocation(), config);
- State state, out;
- FullScoreReturn ret(model.FullScore(model.NullContextState(), model.GetVocabulary().Index("."), state));
- SLOPPY_CHECK_CLOSE(-0.2705918, ret.rest, 0.001);
- SLOPPY_CHECK_CLOSE(-0.01916512, model.FullScore(state, model.GetVocabulary().EndSentence(), out).rest, 0.001);
-}
-
-} // namespace
-} // namespace ngram
-} // namespace lm
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/model_type.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/model_type.hh b/ext/kenlm/lm/model_type.hh
deleted file mode 100644
index dcdc6ac..0000000
--- a/ext/kenlm/lm/model_type.hh
+++ /dev/null
@@ -1,23 +0,0 @@
-#ifndef LM_MODEL_TYPE_H
-#define LM_MODEL_TYPE_H
-
-namespace lm {
-namespace ngram {
-
-/* Not the best numbering system, but it grew this way for historical reasons
- * and I want to preserve existing binary files. */
-typedef enum {PROBING=0, REST_PROBING=1, TRIE=2, QUANT_TRIE=3, ARRAY_TRIE=4, QUANT_ARRAY_TRIE=5} ModelType;
-
-// Historical names.
-const ModelType HASH_PROBING = PROBING;
-const ModelType TRIE_SORTED = TRIE;
-const ModelType QUANT_TRIE_SORTED = QUANT_TRIE;
-const ModelType ARRAY_TRIE_SORTED = ARRAY_TRIE;
-const ModelType QUANT_ARRAY_TRIE_SORTED = QUANT_ARRAY_TRIE;
-
-const static ModelType kQuantAdd = static_cast<ModelType>(QUANT_TRIE - TRIE);
-const static ModelType kArrayAdd = static_cast<ModelType>(ARRAY_TRIE - TRIE);
-
-} // namespace ngram
-} // namespace lm
-#endif // LM_MODEL_TYPE_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/neural/Jamfile
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/neural/Jamfile b/ext/kenlm/lm/neural/Jamfile
deleted file mode 100644
index 14cd8e3..0000000
--- a/ext/kenlm/lm/neural/Jamfile
+++ /dev/null
@@ -1,6 +0,0 @@
-with-eigen = [ option.get "with-eigen" ] ;
-if ! $(with-eigen) && ! [ test_flags "" : "#include <Eigen/Dense>\nint main() {}" ] {
- with-eigen = "/usr/include/eigen3" ;
-}
-with-eigen = <include>$(with-eigen) ;
-fakelib neural : ..//kenlm wordvecs.cc : $(with-eigen) : : <cxxflags>-fopenmp <linkflags>-fopenmp $(with-eigen) ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/neural/wordvecs.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/neural/wordvecs.hh b/ext/kenlm/lm/neural/wordvecs.hh
deleted file mode 100644
index 921a2b2..0000000
--- a/ext/kenlm/lm/neural/wordvecs.hh
+++ /dev/null
@@ -1,38 +0,0 @@
-#ifndef LM_NEURAL_WORDVECS_H
-#define LM_NEURAL_WORDVECS_H
-
-#include "util/scoped.hh"
-#include "lm/vocab.hh"
-
-#include <Eigen/Dense>
-
-namespace util { class FilePiece; }
-
-namespace lm {
-namespace neural {
-
-class WordVecs {
- public:
- // Columns of the matrix are word vectors. The column index is the word.
- typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor> Storage;
-
- /* The file should begin with a line stating the number of word vectors and
- * the length of the vectors. Then it's followed by lines containing a
- * word followed by floating-point values.
- */
- explicit WordVecs(util::FilePiece &in);
-
- const Storage &Vectors() const { return vecs_; }
-
- WordIndex Index(StringPiece str) const { return vocab_.Index(str); }
-
- private:
- util::scoped_malloc vocab_backing_;
- ngram::ProbingVocabulary vocab_;
-
- Storage vecs_;
-};
-
-}} // namespaces
-
-#endif // LM_NEURAL_WORDVECS_H
[45/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/build/project.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/build/project.jam b/ext/kenlm/jam-files/boost-build/build/project.jam
deleted file mode 100644
index c9a0909..0000000
--- a/ext/kenlm/jam-files/boost-build/build/project.jam
+++ /dev/null
@@ -1,1228 +0,0 @@
-# Copyright 2002, 2003 Dave Abrahams
-# Copyright 2002, 2005, 2006 Rene Rivera
-# Copyright 2002, 2003, 2004, 2005, 2006 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# Implements project representation and loading. Each project is represented by:
-# - a module where all the Jamfile content lives.
-# - an instance of 'project-attributes' class.
-# (given a module name, can be obtained using the 'attributes' rule)
-# - an instance of 'project-target' class (from targets.jam)
-# (given a module name, can be obtained using the 'target' rule)
-#
-# Typically, projects are created as result of loading a Jamfile, which is done
-# by rules 'load' and 'initialize', below. First, a module is prepared and a new
-# project-attributes instance is created. Some rules necessary for all projects
-# are added to the module (see the 'project-rules' module). Default project
-# attributes are set (inheriting parent project attributes, if it exists). After
-# that the Jamfile is read. It can declare its own attributes using the
-# 'project' rule which will be combined with any already set.
-#
-# The 'project' rule can also declare a project id which will be associated with
-# the project module.
-#
-# Besides Jamfile projects, we also support 'standalone' projects created by
-# calling 'initialize' in an arbitrary module and not specifying the project's
-# location. After the call, the module can call the 'project' rule, declare main
-# targets and behave as a regular project except that, since it is not
-# associated with any location, it should only declare prebuilt targets.
-#
-# The list of all loaded Jamfiles is stored in the .project-locations variable.
-# It is possible to obtain a module name for a location using the 'module-name'
-# rule. Standalone projects are not recorded and can only be referenced using
-# their project id.
-
-import "class" : new ;
-import modules ;
-import path ;
-import print ;
-import property-set ;
-import sequence ;
-
-
-.debug-loading = [ MATCH ^(--debug-loading)$ : [ modules.peek : ARGV ] ] ;
-
-
-# Loads the Jamfile at the given location. After loading, project global file
-# and Jamfiles needed by the requested one will be loaded recursively. If the
-# Jamfile at that location is loaded already, does nothing. Returns the project
-# module for the Jamfile.
-#
-rule load ( jamfile-location )
-{
- local module-name = [ module-name $(jamfile-location) ] ;
- # If Jamfile is already loaded, do not try again.
- if ! $(module-name) in $(.jamfile-modules)
- {
- if $(.debug-loading)
- {
- ECHO Loading Jamfile at '$(jamfile-location)' ;
- }
-
- load-jamfile $(jamfile-location) : $(module-name) ;
-
- # We want to make sure that child project are loaded only after parent
- # projects. In particular, because parent projects define attributes
- # which are then inherited by children, and we do not want children to
- # be loaded before parent has defined everything.
- #
- # While "build-project" and "use-project" can potentially refer to child
- # projects from parent projects, we do not immediately load child
- # projects when seeing those attributes. Instead, we record the minimal
- # information to be used only later.
- load-used-projects $(module-name) ;
- }
- return $(module-name) ;
-}
-
-
-rule load-used-projects ( module-name )
-{
- local used = [ modules.peek $(module-name) : .used-projects ] ;
- local location = [ attribute $(module-name) location ] ;
- while $(used)
- {
- local id = $(used[1]) ;
- local where = [ path.make $(used[2]) ] ;
- register-id $(id) : [ load [ path.root $(where) $(location) ] ] ;
- used = $(used[3-]) ;
- }
-}
-
-
-# Note the use of character groups, as opposed to listing 'Jamroot' and
-# 'jamroot'. With the latter, we would get duplicate matches on Windows and
-# would have to eliminate duplicates.
-JAMROOT ?= [ modules.peek : JAMROOT ] ;
-JAMROOT ?= project-root.jam [Jj]amroot [Jj]amroot.jam ;
-
-
-# Loads parent of Jamfile at 'location'. Issues an error if nothing is found.
-#
-rule load-parent ( location )
-{
- local found = [ path.glob-in-parents $(location) : $(JAMROOT) $(JAMFILE) ] ;
- if ! $(found)
- {
- import errors ;
- errors.error Could not find parent "for" project at '$(location)' :
- Did not find Jamfile.jam or Jamroot.jam "in" any parent directory. ;
- }
- return [ load $(found[1]:D) ] ;
-}
-
-
-# Returns the project module corresponding to the given project-id or plain
-# directory name. Returns nothing if such a project can not be found.
-#
-rule find ( name : current-location )
-{
- local project-module ;
-
- # Try interpreting name as project id.
- if [ path.is-rooted $(name) ]
- {
- project-module = $($(name).jamfile-module) ;
- }
-
- if ! $(project-module)
- {
- local location = [ path.root [ path.make $(name) ] $(current-location) ]
- ;
-
- # If no project is registered for the given location, try to load it.
- # First see if we have a Jamfile. If not, then see if we might have a
- # project root willing to act as a Jamfile. In that case, project root
- # must be placed in the directory referred to by id.
-
- project-module = [ module-name $(location) ] ;
- if ! $(project-module) in $(.jamfile-modules)
- {
- if [ path.glob $(location) : $(JAMROOT) $(JAMFILE) ]
- {
- project-module = [ load $(location) ] ;
- }
- else
- {
- project-module = ;
- }
- }
- }
-
- return $(project-module) ;
-}
-
-
-# Returns the name of the module corresponding to 'jamfile-location'. If no
-# module corresponds to that location yet, associates the default module name
-# with that location.
-#
-rule module-name ( jamfile-location )
-{
- if ! $(.module.$(jamfile-location))
- {
- # Root the path, so that locations are always unambiguous. Without this,
- # we can not decide if '../../exe/program1' and '.' are the same paths.
- local normalized = [ path.root $(jamfile-location) [ path.pwd ] ] ;
-
- # Quick & dirty fix to get the same module name when we supply two
- # equivalent location paths, e.g. 'd:\Foo' & 'D:\fOo\bar\..' on Windows.
- # Note that our current implementation will not work correctly if the
- # given location references an empty folder, but in that case any later
- # attempt to load a Jamfile from this location will fail anyway.
- # FIXME: Implement this cleanly. Support for this type of path
- # normalization already exists internally in Boost Jam and the current
- # fix relies on the GLOB builtin rule using that support. Most likely we
- # just need to add a new builtin rule to do this explicitly.
- normalized = [ NORMALIZE_PATH $(normalized) ] ;
- local glob-result = [ GLOB [ path.native $(normalized) ] : * ] ;
- if $(glob-result)
- {
- normalized = $(glob-result[1]:D) ;
- }
- .module.$(jamfile-location) = Jamfile<$(normalized)> ;
- }
- return $(.module.$(jamfile-location)) ;
-}
-
-
-# Default patterns to search for the Jamfiles to use for build declarations.
-#
-JAMFILE = [ modules.peek : JAMFILE ] ;
-JAMFILE ?= [Bb]uild.jam [Jj]amfile.v2 [Jj]amfile [Jj]amfile.jam ;
-
-
-# Find the Jamfile at the given location. This returns the exact names of all
-# the Jamfiles in the given directory. The optional parent-root argument causes
-# this to search not the given directory but the ones above it up to the
-# parent-root directory.
-#
-rule find-jamfile (
- dir # The directory(s) to look for a Jamfile.
- parent-root ? # Optional flag indicating to search for the parent Jamfile.
- : no-errors ?
- )
-{
- # Glob for all the possible Jamfiles according to the match pattern.
- #
- local jamfile-glob = ;
- if $(parent-root)
- {
- if ! $(.parent-jamfile.$(dir))
- {
- .parent-jamfile.$(dir) = [ path.glob-in-parents $(dir) : $(JAMFILE)
- ] ;
- }
- jamfile-glob = $(.parent-jamfile.$(dir)) ;
- }
- else
- {
- if ! $(.jamfile.$(dir))
- {
- .jamfile.$(dir) = [ path.glob $(dir) : $(JAMFILE) ] ;
- }
- jamfile-glob = $(.jamfile.$(dir)) ;
-
- }
-
- local jamfile-to-load = $(jamfile-glob) ;
- # Multiple Jamfiles found in the same place. Warn about this and ensure we
- # use only one of them. As a temporary convenience measure, if there is
- # Jamfile.v2 among found files, suppress the warning and use it.
- #
- if $(jamfile-to-load[2-])
- {
- local v2-jamfiles = [ MATCH ^(.*[Jj]amfile\\.v2)|(.*[Bb]uild\\.jam)$ :
- $(jamfile-to-load) ] ;
-
- if $(v2-jamfiles) && ! $(v2-jamfiles[2])
- {
- jamfile-to-load = $(v2-jamfiles) ;
- }
- else
- {
- local jamfile = [ path.basename $(jamfile-to-load[1]) ] ;
- ECHO "warning: Found multiple Jamfiles at '"$(dir)"'!"
- "Loading the first one: '$(jamfile)'." ;
- }
-
- jamfile-to-load = $(jamfile-to-load[1]) ;
- }
-
- # Could not find it, error.
- #
- if ! $(no-errors) && ! $(jamfile-to-load)
- {
- import errors ;
- errors.error Unable to load Jamfile.
- : Could not find a Jamfile in directory '$(dir)'.
- : Attempted to find it with pattern '$(JAMFILE:J=" ")'.
- : Please consult the documentation at 'http://www.boost.org'. ;
- }
-
- return $(jamfile-to-load) ;
-}
-
-
-# Load a Jamfile at the given directory. Returns nothing. Will attempt to load
-# the file as indicated by the JAMFILE patterns. Effect of calling this rule
-# twice with the same 'dir' is undefined.
-#
-local rule load-jamfile ( dir : jamfile-module )
-{
- # See if the Jamfile is where it should be.
- #
- local jamfile-to-load = [ path.glob $(dir) : $(JAMROOT) ] ;
- if ! $(jamfile-to-load)
- {
- jamfile-to-load = [ find-jamfile $(dir) ] ;
- }
-
- if $(jamfile-to-load[2])
- {
- import errors ;
- errors.error "Multiple Jamfiles found at '$(dir)'" :
- "Filenames are: " $(jamfile-to-load:D=) ;
- }
-
- # Now load the Jamfile in its own context.
- # The call to 'initialize' may load the parent Jamfile, which might contain
- # a 'use-project' or a 'project.load' call, causing a second attempt to load
- # the same project we are loading now. Checking inside .jamfile-modules
- # prevents that second attempt from messing things up.
- if ! $(jamfile-module) in $(.jamfile-modules)
- {
- local previous-project = $(.current-project) ;
-
- # Initialize the Jamfile module before loading.
- initialize $(jamfile-module) : [ path.parent $(jamfile-to-load) ] :
- $(jamfile-to-load:BS) ;
-
- if ! $(jamfile-module) in $(.jamfile-modules)
- {
- .jamfile-modules += $(jamfile-module) ;
-
- local saved-project = $(.current-project) ;
-
- mark-as-user $(jamfile-module) ;
- modules.load $(jamfile-module) : [ path.native $(jamfile-to-load) ]
- : . ;
- if [ MATCH ^($(JAMROOT))$ : $(jamfile-to-load:BS) ]
- {
- jamfile = [ find-jamfile $(dir) : no-errors ] ;
- if $(jamfile)
- {
- load-aux $(jamfile-module) : [ path.native $(jamfile) ] ;
- }
- }
-
- # Now do some checks.
- if $(.current-project) != $(saved-project)
- {
- import errors ;
- errors.error
- The value of the .current-project variable has magically
- : changed after loading a Jamfile. This means some of the
- : targets might be defined in the wrong project.
- : after loading $(jamfile-module)
- : expected value $(saved-project)
- : actual value $(.current-project) ;
- }
-
- end-load $(previous-project) ;
-
- if $(.global-build-dir)
- {
- if [ attribute $(jamfile-module) location ] && ! [ attribute
- $(jamfile-module) id ]
- {
- local project-root = [ attribute $(jamfile-module)
- project-root ] ;
- if $(project-root) = $(dir)
- {
- ECHO "warning: the --build-dir option was specified" ;
- ECHO "warning: but Jamroot at '$(dir)'" ;
- ECHO "warning: specified no project id" ;
- ECHO "warning: the --build-dir option will be ignored" ;
- }
- }
- }
- }
- }
-}
-
-
-# Called when done loading a project module. Restores the current project to its
-# previous value and does some additional checking to make sure our 'currently
-# loaded project' identifier does not get left with an invalid value.
-#
-rule end-load ( previous-project ? )
-{
- if ! $(.current-project)
- {
- import errors ;
- errors.error Ending project loading requested when there was no project
- currently being loaded. ;
- }
-
- if ! $(previous-project) && $(.saved-current-project)
- {
- import errors ;
- errors.error Ending project loading requested with no 'previous project'
- when there were other projects still marked as being loaded
- recursively. ;
- }
-
- .current-project = $(previous-project) ;
-}
-
-
-rule mark-as-user ( module-name )
-{
- if USER_MODULE in [ RULENAMES ]
- {
- USER_MODULE $(module-name) ;
- }
-}
-
-
-rule load-aux ( module-name : file )
-{
- mark-as-user $(module-name) ;
-
- module $(module-name)
- {
- include $(2) ;
- local rules = [ RULENAMES $(1) ] ;
- IMPORT $(1) : $(rules) : $(1) : $(1).$(rules) ;
- }
-}
-
-
-.global-build-dir = [ MATCH ^--build-dir=(.*)$ : [ modules.peek : ARGV ] ] ;
-if $(.global-build-dir)
-{
- # If the option is specified several times, take the last value.
- .global-build-dir = [ path.make $(.global-build-dir[-1]) ] ;
-}
-
-
-# Initialize the module for a project.
-#
-rule initialize (
- module-name # The name of the project module.
- : location ? # The location (directory) of the project to initialize. If
- # not specified, a standalone project will be initialized.
- : basename ?
- )
-{
- if $(.debug-loading)
- {
- ECHO "Initializing project '$(module-name)'" ;
- }
-
- local jamroot ;
-
- local parent-module ;
- if $(module-name) = test-config
- {
- # No parent.
- }
- else if $(module-name) = site-config
- {
- parent-module = test-config ;
- }
- else if $(module-name) = user-config
- {
- parent-module = site-config ;
- }
- else if $(module-name) = project-config
- {
- parent-module = user-config ;
- }
- else if $(location) && ! [ MATCH ^($(JAMROOT))$ : $(basename) ]
- {
- # We search for parent/jamroot only if this is a jamfile project, i.e.
- # if is not a standalone or a jamroot project.
- parent-module = [ load-parent $(location) ] ;
- }
- else if $(location)
- {
- # We have a jamroot project. Inherit from user-config (or project-config
- # if it exists).
- if $(project-config.attributes)
- {
- parent-module = project-config ;
- }
- else
- {
- parent-module = user-config ;
- }
- jamroot = true ;
- }
-
- # TODO: need to consider if standalone projects can do anything but define
- # prebuilt targets. If so, we need to give them a more sensible "location",
- # so that source paths are correct.
- location ?= "" ;
- # Create the module for the Jamfile first.
- module $(module-name)
- {
- }
-
- # load-parent can end up loading this module again. Make sure this is not
- # duplicated.
- if ! $($(module-name).attributes)
- {
- $(module-name).attributes = [ new project-attributes $(location)
- $(module-name) ] ;
- local attributes = $($(module-name).attributes) ;
-
- if $(location)
- {
- $(attributes).set source-location : [ path.make $(location) ] :
- exact ;
- }
- else
- {
- local cfgs = project site test user ;
- if ! $(module-name) in $(cfgs)-config
- {
- # This is a standalone project with known location. Set its
- # source location so it can declare targets. This is needed so
- # you can put a .jam file with your sources and use it via
- # 'using'. Standard modules (in the 'tools' subdir) may not
- # assume source dir is set.
- local s = [ modules.binding $(module-name) ] ;
- if ! $(s)
- {
- import errors ;
- errors.error Could not determine project location
- $(module-name) ;
- }
- $(attributes).set source-location : $(s:D) : exact ;
- }
- }
-
- $(attributes).set requirements : [ property-set.empty ] : exact ;
- $(attributes).set usage-requirements : [ property-set.empty ] : exact ;
-
- # Import rules common to all project modules from project-rules module,
- # defined at the end of this file.
- local rules = [ RULENAMES project-rules ] ;
- IMPORT project-rules : $(rules) : $(module-name) : $(rules) ;
-
- if $(parent-module)
- {
- inherit-attributes $(module-name) : $(parent-module) ;
- $(attributes).set parent-module : $(parent-module) : exact ;
- }
-
- if $(jamroot)
- {
- $(attributes).set project-root : $(location) : exact ;
- if ! $(.first-project-root)
- {
- .first-project-root = $(module-name) ;
- }
- }
-
- local parent ;
- if $(parent-module)
- {
- parent = [ target $(parent-module) ] ;
- }
-
- if ! $(.target.$(module-name))
- {
- local requirements = [ attribute $(module-name) requirements ] ;
- .target.$(module-name) = [ new project-target $(module-name) :
- $(module-name) $(parent) : $(requirements) ] ;
-
- if $(.debug-loading)
- {
- ECHO Assigned project target $(.target.$(module-name)) to
- '$(module-name)' ;
- }
- }
- }
-
- .current-project = [ target $(module-name) ] ;
-}
-
-
-# Make 'project-module' inherit attributes of project root and parent module.
-#
-rule inherit-attributes ( project-module : parent-module )
-{
- local attributes = $($(project-module).attributes) ;
- local pattributes = [ attributes $(parent-module) ] ;
- # Parent module might be locationless configuration module.
- if [ modules.binding $(parent-module) ]
- {
- $(attributes).set parent :
- [ path.parent [ path.make [ modules.binding $(parent-module) ] ] ] ;
- }
- $(attributes).set project-root :
- [ $(pattributes).get project-root ] : exact ;
- $(attributes).set default-build :
- [ $(pattributes).get default-build ] ;
- $(attributes).set requirements :
- [ $(pattributes).get requirements ] : exact ;
- $(attributes).set usage-requirements :
- [ $(pattributes).get usage-requirements ] : exact ;
-
- local parent-build-dir = [ $(pattributes).get build-dir ] ;
- if $(parent-build-dir)
- {
- # Have to compute relative path from parent dir to our dir. Convert both
- # paths to absolute, since we cannot find relative path from ".." to
- # ".".
-
- local location = [ attribute $(project-module) location ] ;
- local parent-location = [ attribute $(parent-module) location ] ;
-
- local pwd = [ path.pwd ] ;
- local parent-dir = [ path.root $(parent-location) $(pwd) ] ;
- local our-dir = [ path.root $(location) $(pwd) ] ;
- $(attributes).set build-dir : [ path.join $(parent-build-dir)
- [ path.relative $(our-dir) $(parent-dir) ] ] : exact ;
- }
-}
-
-
-# Returns whether the given string is a valid registered project id.
-#
-rule is-registered-id ( id )
-{
- return $($(id).jamfile-module) ;
-}
-
-
-# Associate the given id with the given project module. Returns the possibly
-# corrected project id.
-#
-rule register-id ( id : module )
-{
- id = [ path.root $(id) / ] ;
-
- if [ MATCH (//) : $(id) ]
- {
- import errors ;
- errors.user-error Project id may not contain two consecutive slash
- characters (project id: '$(id)'). ;
- }
-
- local orig-module = $($(id).jamfile-module) ;
- if $(orig-module) && $(orig-module) != $(module)
- {
- local new-file = [ modules.peek $(module) : __file__ ] ;
- local new-location = [ project.attribute $(module) location ] ;
-
- local orig-file = [ modules.peek $(orig-module) : __file__ ] ;
- local orig-main-id = [ project.attribute $(orig-module) id ] ;
- local orig-location = [ project.attribute $(orig-module) location ] ;
- local orig-project = [ target $(orig-module) ] ;
- local orig-name = [ $(orig-project).name ] ;
-
- import errors ;
- errors.user-error Attempt to redeclare already registered project id
- '$(id)'.
- : Original project:
- : " " Name: $(orig-name:E=---)
- : " " Module: $(orig-module)
- : " " Main id: $(orig-main-id:E=---)
- : " " File: $(orig-file:E=---)
- : " " Location: $(orig-location:E=---)
- : New project:
- : " " Module: $(module)
- : " " File: $(new-file:E=---)
- : " " Location: $(new-location:E=---) ;
- }
-
- $(id).jamfile-module = $(module) ;
- return $(id) ;
-}
-
-
-# Class keeping all the attributes of a project.
-#
-# The standard attributes are "id", "location", "project-root", "parent"
-# "requirements", "default-build", "source-location" and "projects-to-build".
-#
-class project-attributes
-{
- import path ;
- import print ;
- import project ;
- import property ;
- import property-set ;
- import sequence ;
-
- rule __init__ ( location project-module )
- {
- self.location = $(location) ;
- self.project-module = $(project-module) ;
- }
-
- # Set the named attribute from the specification given by the user. The
- # value actually set may be different.
- #
- rule set ( attribute : specification *
- : exact ? # Sets value from 'specification' without any processing.
- )
- {
- if $(exact)
- {
- self.$(attribute) = $(specification) ;
- }
- else if $(attribute) = "requirements"
- {
- local result = [ property-set.refine-from-user-input
- $(self.requirements) : $(specification)
- : $(self.project-module) : $(self.location) ] ;
-
- if $(result[1]) = "@error"
- {
- import errors : error : errors.error ;
- errors.error Requirements for project at '$(self.location)'
- conflict with parent's. : Explanation: $(result[2-]) ;
- }
-
- self.requirements = $(result) ;
- }
- else if $(attribute) = "usage-requirements"
- {
- local unconditional ;
- for local p in $(specification)
- {
- local split = [ property.split-conditional $(p) ] ;
- split ?= nothing $(p) ;
- unconditional += $(split[2]) ;
- }
-
- local non-free = [ property.remove free : $(unconditional) ] ;
- if $(non-free)
- {
- import errors : error : errors.error ;
- errors.error usage-requirements $(specification) have non-free
- properties $(non-free) ;
- }
- local t = [ property.translate-paths $(specification) :
- $(self.location) ] ;
- if $(self.usage-requirements)
- {
- self.usage-requirements = [ property-set.create
- [ $(self.usage-requirements).raw ] $(t) ] ;
- }
- else
- {
- self.usage-requirements = [ property-set.create $(t) ] ;
- }
- }
- else if $(attribute) = "default-build"
- {
- self.default-build = [ property.make $(specification) ] ;
- }
- else if $(attribute) = "source-location"
- {
- self.source-location = ;
- for local src-path in $(specification)
- {
- self.source-location += [ path.root [ path.make $(src-path) ]
- $(self.location) ] ;
- }
- }
- else if $(attribute) = "build-dir"
- {
- self.build-dir = [ path.root [ path.make $(specification) ]
- $(self.location) ] ;
- }
- else if $(attribute) = "id"
- {
- self.id = [ project.register-id $(specification) :
- $(self.project-module) ] ;
- }
- else if ! $(attribute) in "default-build" "location" "parent"
- "projects-to-build" "project-root" "source-location"
- {
- import errors : error : errors.error ;
- errors.error Invalid project attribute '$(attribute)' specified for
- project at '$(self.location)' ;
- }
- else
- {
- self.$(attribute) = $(specification) ;
- }
- }
-
- # Returns the value of the given attribute.
- #
- rule get ( attribute )
- {
- return $(self.$(attribute)) ;
- }
-
- # Returns whether these attributes belong to a Jamroot project module.
- #
- rule is-jamroot ( )
- {
- if $(self.location) && $(self.project-root) = $(self.location)
- {
- return true ;
- }
- }
-
- # Prints the project attributes.
- #
- rule print ( )
- {
- local id = '$(self.id)' ;
- print.section $(id:E=(none)) ;
- print.list-start ;
- print.list-item "Parent project:" $(self.parent:E=(none)) ;
- print.list-item "Requirements:" [ $(self.requirements).raw ] ;
- print.list-item "Default build:" $(self.default-build) ;
- print.list-item "Source location:" $(self.source-location) ;
- print.list-item "Projects to build:" [ sequence.insertion-sort
- $(self.projects-to-build) ] ;
- print.list-end ;
- }
-}
-
-
-# Returns the build directory for standalone projects
-#
-rule standalone-build-dir ( )
-{
- project = [ target $(.first-project-root) ] ;
- return [ path.join [ $(project).build-dir ] standalone ] ;
-}
-
-# Returns the project which is currently being loaded.
-#
-rule current ( )
-{
- if ! $(.current-project)
- {
- import errors ;
- errors.error Reference to the project currently being loaded requested
- when there was no project module being loaded. ;
- }
- return $(.current-project) ;
-}
-
-
-# Temporarily changes the current project to 'project'. Should be followed by
-# 'pop-current'.
-#
-rule push-current ( project )
-{
- .saved-current-project += $(.current-project) ;
- .current-project = $(project) ;
-}
-
-
-rule pop-current ( )
-{
- .current-project = $(.saved-current-project[-1]) ;
- .saved-current-project = $(.saved-current-project[1--2]) ;
-}
-
-
-# Returns the project-attribute instance for the specified Jamfile module.
-#
-rule attributes ( project )
-{
- return $($(project).attributes) ;
-}
-
-
-# Returns the value of the specified attribute in the specified Jamfile module.
-#
-rule attribute ( project attribute )
-{
- return [ $($(project).attributes).get $(attribute) ] ;
-}
-
-
-# Returns whether a project module is one of Boost Build's configuration
-# modules.
-#
-rule is-config-module ( project )
-{
- local cfgs = project site test user ;
- if $(project) in $(cfgs)-config
- {
- return true ;
- }
-}
-
-
-# Returns whether a project module is a Jamroot project module.
-#
-rule is-jamroot-module ( project )
-{
- return [ $($(project).attributes).is-jamroot ] ;
-}
-
-
-# Returns a project's parent jamroot module. Returns nothing if there is no such
-# module, i.e. if this is a standalone project or one of the internal Boost
-# Build configuration projects.
-#
-rule get-jamroot-module ( project )
-{
- local jamroot-location = [ attribute $(project) project-root ] ;
- if $(jamroot-location)
- {
- return [ module-name $(jamroot-location) ] ;
- }
-}
-
-
-# Returns the project target corresponding to the 'project-module'.
-#
-rule target ( project-module )
-{
- if ! $(.target.$(project-module))
- {
- import errors ;
- errors.user-error Project target requested but not yet assigned for
- module '$(project-module)'. ;
- }
- return $(.target.$(project-module)) ;
-}
-
-
-# Defines a Boost.Build extension project. Such extensions usually contain
-# library targets and features that can be used by many people. Even though
-# extensions are really projects, they can be initialized as a module would be
-# with the "using" (project.project-rules.using) mechanism.
-#
-rule extension ( id : options * : * )
-{
- # The caller is a standalone module for the extension.
- local mod = [ CALLER_MODULE ] ;
-
- # We need to do the rest within the extension module.
- module $(mod)
- {
- import path ;
-
- # Find the root project.
- local root-project = [ project.current ] ;
- root-project = [ $(root-project).project-module ] ;
- while
- [ project.attribute $(root-project) parent-module ] &&
- [ project.attribute $(root-project) parent-module ] != user-config
- {
- root-project = [ project.attribute $(root-project) parent-module ] ;
- }
-
- # Create the project data, and bring in the project rules into the
- # module.
- project.initialize $(__name__) : [ path.join [ project.attribute
- $(root-project) location ] ext $(1:L) ] ;
-
- # Create the project itself, i.e. the attributes. All extensions are
- # created in the "/ext" project space.
- project /ext/$(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) :
- $(9) : $(10) : $(11) : $(12) : $(13) : $(14) : $(15) : $(16) : $(17)
- : $(18) : $(19) ;
- local attributes = [ project.attributes $(__name__) ] ;
-
- # Inherit from the root project of whomever is defining us.
- project.inherit-attributes $(__name__) : $(root-project) ;
- $(attributes).set parent-module : $(root-project) : exact ;
- }
-}
-
-
-rule glob-internal ( project : wildcards + : excludes * : rule-name )
-{
- local location = [ $(project).get source-location ] ;
-
- local result ;
- local paths = [ path.$(rule-name) $(location) :
- [ sequence.transform path.make : $(wildcards) ] :
- [ sequence.transform path.make : $(excludes) ] ] ;
- if $(wildcards:D) || $(rule-name) != glob
- {
- # The paths we have found are relative to the current directory, but the
- # names specified in the sources list are assumed to be relative to the
- # source directory of the corresponding project. So, just make the names
- # absolute.
- for local p in $(paths)
- {
- # If the path is below source location, use relative path.
- # Otherwise, use full path just to avoid any ambiguities.
- local rel = [ path.relative $(p) $(location) : no-error ] ;
- if $(rel) = not-a-child
- {
- result += [ path.root $(p) [ path.pwd ] ] ;
- }
- else
- {
- result += $(rel) ;
- }
- }
- }
- else
- {
- # There were no wildcards in the directory path, so the files are all in
- # the source directory of the project. Just drop the directory, instead
- # of making paths absolute.
- result = $(paths:D="") ;
- }
-
- return $(result) ;
-}
-
-
-# This module defines rules common to all projects.
-#
-module project-rules
-{
- import modules ;
-
- rule using ( toolset-module : * )
- {
- import toolset ;
-
- local saved-project = [ modules.peek project : .current-project ] ;
-
- # Temporarily change the search path so the module referred to by
- # 'using' can be placed in the same directory as Jamfile. User will
- # expect the module to be found even though the directory is not in
- # BOOST_BUILD_PATH.
- local x = [ modules.peek : BOOST_BUILD_PATH ] ;
- local caller = [ CALLER_MODULE ] ;
- local caller-location = [ modules.binding $(caller) ] ;
- modules.poke : BOOST_BUILD_PATH : $(caller-location:D) $(x) ;
- toolset.using $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) :
- $(9) : $(10) : $(11) : $(12) : $(13) : $(14) : $(15) : $(16) : $(17)
- : $(18) : $(19) ;
- modules.poke : BOOST_BUILD_PATH : $(x) ;
-
- # The above might have clobbered .current-project in case it caused a
- # new project instance to be created (which would then automatically
- # get set as the 'current' project). Restore the correct value so any
- # main targets declared after this do not get mapped to the loaded
- # module's project.
- modules.poke project : .current-project : $(saved-project) ;
- }
-
- rule import ( * : * : * )
- {
- local caller = [ CALLER_MODULE ] ;
- local saved-project = [ modules.peek project : .current-project ] ;
- module $(caller)
- {
- modules.import $(1) : $(2) : $(3) ;
- }
-
- # The above might have clobbered .current-project in case it caused a
- # new project instance to be created (which would then automatically
- # get set as the 'current' project). Restore the correct value so any
- # main targets declared after this do not get mapped to the loaded
- # module's project.
- modules.poke project : .current-project : $(saved-project) ;
- }
-
- rule project ( id ? : options * : * )
- {
- import path ;
- import project ;
-
- local caller = [ CALLER_MODULE ] ;
- local attributes = [ project.attributes $(caller) ] ;
- if $(id)
- {
- $(attributes).set id : $(id) ;
- }
-
- local explicit-build-dir ;
-
- for n in 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
- {
- local option = $($(n)) ;
- if $(option)
- {
- $(attributes).set $(option[1]) : $(option[2-]) ;
- }
- if $(option[1]) = "build-dir"
- {
- explicit-build-dir = [ path.make $(option[2-]) ] ;
- }
- }
-
- # If '--build-dir' is specified, change the build dir for the project.
- local global-build-dir = [ modules.peek project : .global-build-dir ] ;
-
- if $(global-build-dir)
- {
- local location = [ $(attributes).get location ] ;
- # Project with an empty location is a 'standalone' project such as
- # user-config or qt. It has no build dir. If we try to set build dir
- # for user-config, we shall then try to inherit it, with either
- # weird or wrong consequences.
- if $(location) && $(location) = [ $(attributes).get project-root ]
- {
- # Re-read the project id, since it might have been modified a
- # bit when setting the project's id attribute, e.g. might have
- # been prefixed by a slash if it was not already.
- id = [ $(attributes).get id ] ;
- # This is Jamroot.
- if $(id)
- {
- if $(explicit-build-dir) &&
- [ path.is-rooted $(explicit-build-dir) ]
- {
- import errors ;
- errors.user-error Absolute directory specified via
- 'build-dir' project attribute : Do not know how to
- combine that with the --build-dir option. ;
- }
- # Strip the leading slash from id.
- local rid = [ MATCH ^/(.*) : $(id) ] ;
- local p = [ path.join $(global-build-dir) $(rid)
- $(explicit-build-dir) ] ;
-
- $(attributes).set build-dir : $(p) : exact ;
- }
- }
- else
- {
- # Not Jamroot.
- if $(explicit-build-dir)
- {
- import errors ;
- errors.user-error When --build-dir is specified, the
- 'build-dir' project : attribute is allowed only for
- top-level 'project' invocations ;
- }
- }
- }
- }
-
- # Declare and set a project global constant. Project global constants are
- # normal variables but should not be changed. They are applied to every
- # child Jamfile.
- #
- rule constant ( name : value + )
- {
- import project ;
- local caller = [ CALLER_MODULE ] ;
- local p = [ project.target $(caller) ] ;
- $(p).add-constant $(name) : $(value) ;
- }
-
- # Declare and set a project global constant, whose value is a path. The path
- # is adjusted to be relative to the invocation directory. The given value
- # path is taken to be either absolute, or relative to this project root.
- #
- rule path-constant ( name : value + )
- {
- import project ;
- local caller = [ CALLER_MODULE ] ;
- local p = [ project.target $(caller) ] ;
- $(p).add-constant $(name) : $(value) : path ;
- }
-
- rule use-project ( id : where )
- {
- # See comment in 'load' for explanation.
- local caller = [ CALLER_MODULE ] ;
- modules.poke $(caller) : .used-projects : [ modules.peek $(caller) :
- .used-projects ] $(id) $(where) ;
- }
-
- rule build-project ( dir )
- {
- import project ;
- local caller = [ CALLER_MODULE ] ;
- local attributes = [ project.attributes $(caller) ] ;
- local now = [ $(attributes).get projects-to-build ] ;
- $(attributes).set projects-to-build : $(now) $(dir) ;
- }
-
- rule explicit ( target-names * )
- {
- import project ;
- # If 'explicit' is used in a helper rule defined in Jamroot and
- # inherited by children, then most of the time we want 'explicit' to
- # operate on the Jamfile where the helper rule is invoked.
- local t = [ project.current ] ;
- for local n in $(target-names)
- {
- $(t).mark-target-as-explicit $(n) ;
- }
- }
-
- rule always ( target-names * )
- {
- import project ;
- local t = [ project.current ] ;
- for local n in $(target-names)
- {
- $(t).mark-target-as-always $(n) ;
- }
- }
-
- rule glob ( wildcards + : excludes * )
- {
- import project ;
- return [ project.glob-internal [ project.current ] : $(wildcards) :
- $(excludes) : glob ] ;
- }
-
- rule glob-tree ( wildcards + : excludes * )
- {
- import project ;
- if $(wildcards:D) || $(excludes:D)
- {
- import errors ;
- errors.user-error The patterns to 'glob-tree' may not include
- directory ;
- }
- return [ project.glob-internal [ project.current ] : $(wildcards) :
- $(excludes) : glob-tree ] ;
- }
-
- # Calculates conditional requirements for multiple requirements at once.
- # This is a shorthand to reduce duplication and to keep an inline
- # declarative syntax. For example:
- #
- # lib x : x.cpp : [ conditional <toolset>gcc <variant>debug :
- # <define>DEBUG_EXCEPTION <define>DEBUG_TRACE ] ;
- #
- rule conditional ( condition + : requirements * )
- {
- local condition = $(condition:J=,) ;
- if [ MATCH (:) : $(condition) ]
- {
- return $(condition)$(requirements) ;
- }
- else
- {
- return $(condition):$(requirements) ;
- }
- }
-
- rule option ( name : value )
- {
- local m = [ CALLER_MODULE ] ;
- local cfgs = project site test user ;
- if ! $(m) in $(cfgs)-config
- {
- import errors ;
- errors.error The 'option' rule may only be used "in" Boost Build
- configuration files. ;
- }
- import option ;
- option.set $(name) : $(value) ;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/build/property-set.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/build/property-set.jam b/ext/kenlm/jam-files/boost-build/build/property-set.jam
deleted file mode 100644
index 55cb556..0000000
--- a/ext/kenlm/jam-files/boost-build/build/property-set.jam
+++ /dev/null
@@ -1,517 +0,0 @@
-# Copyright 2003 Dave Abrahams
-# Copyright 2003, 2004, 2005, 2006 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-import "class" : new ;
-import feature ;
-import path ;
-import project ;
-import property ;
-import sequence ;
-import set ;
-import option ;
-
-# Class for storing a set of properties.
-#
-# There is 1<->1 correspondence between identity and value. No two instances
-# of the class are equal. To maintain this property, the 'property-set.create'
-# rule should be used to create new instances. Instances are immutable.
-#
-# Each property is classified with regard to its effect on build results.
-# Incidental properties have no effect on build results, from Boost.Build's
-# point of view. Others are either free, or non-free and we refer to non-free
-# ones as 'base'. Each property belongs to exactly one of those categories.
-#
-# It is possible to get a list of properties belonging to each category as
-# well as a list of properties with a specific attribute.
-#
-# Several operations, like and refine and as-path are provided. They all use
-# caching whenever possible.
-#
-class property-set
-{
- import errors ;
- import feature ;
- import path ;
- import property ;
- import property-set ;
- import set ;
-
- rule __init__ ( raw-properties * )
- {
- self.raw = $(raw-properties) ;
-
- for local p in $(raw-properties)
- {
- if ! $(p:G)
- {
- errors.error "Invalid property: '$(p)'" ;
- }
- }
- }
-
- # Returns Jam list of stored properties.
- #
- rule raw ( )
- {
- return $(self.raw) ;
- }
-
- rule str ( )
- {
- return "[" $(self.raw) "]" ;
- }
-
- # Returns properties that are neither incidental nor free.
- #
- rule base ( )
- {
- if ! $(self.base-initialized)
- {
- init-base ;
- }
- return $(self.base) ;
- }
-
- # Returns free properties which are not incidental.
- #
- rule free ( )
- {
- if ! $(self.base-initialized)
- {
- init-base ;
- }
- return $(self.free) ;
- }
-
- # Returns dependency properties.
- #
- rule dependency ( )
- {
- if ! $(self.dependency-initialized)
- {
- init-dependency ;
- }
- return $(self.dependency) ;
- }
-
- rule non-dependency ( )
- {
- if ! $(self.dependency-initialized)
- {
- init-dependency ;
- }
- return $(self.non-dependency) ;
- }
-
- rule conditional ( )
- {
- if ! $(self.conditional-initialized)
- {
- init-conditional ;
- }
- return $(self.conditional) ;
- }
-
- rule non-conditional ( )
- {
- if ! $(self.conditional-initialized)
- {
- init-conditional ;
- }
- return $(self.non-conditional) ;
- }
-
- # Returns incidental properties.
- #
- rule incidental ( )
- {
- if ! $(self.base-initialized)
- {
- init-base ;
- }
- return $(self.incidental) ;
- }
-
- rule refine ( ps )
- {
- if ! $(self.refined.$(ps))
- {
- local r = [ property.refine $(self.raw) : [ $(ps).raw ] ] ;
- if $(r[1]) != "@error"
- {
- self.refined.$(ps) = [ property-set.create $(r) ] ;
- }
- else
- {
- self.refined.$(ps) = $(r) ;
- }
- }
- return $(self.refined.$(ps)) ;
- }
-
- rule expand ( )
- {
- if ! $(self.expanded)
- {
- self.expanded = [ property-set.create [ feature.expand $(self.raw) ]
- ] ;
- }
- return $(self.expanded) ;
- }
-
- rule expand-composites ( )
- {
- if ! $(self.composites)
- {
- self.composites = [ property-set.create
- [ feature.expand-composites $(self.raw) ] ] ;
- }
- return $(self.composites) ;
- }
-
- rule evaluate-conditionals ( context ? )
- {
- context ?= $(__name__) ;
- if ! $(self.evaluated.$(context))
- {
- self.evaluated.$(context) = [ property-set.create
- [ property.evaluate-conditionals-in-context $(self.raw) : [
- $(context).raw ] ] ] ;
- }
- return $(self.evaluated.$(context)) ;
- }
-
- rule propagated ( )
- {
- if ! $(self.propagated-ps)
- {
- local result ;
- for local p in $(self.raw)
- {
- if propagated in [ feature.attributes $(p:G) ]
- {
- result += $(p) ;
- }
- }
- self.propagated-ps = [ property-set.create $(result) ] ;
- }
- return $(self.propagated-ps) ;
- }
-
- rule add-defaults ( )
- {
- if ! $(self.defaults)
- {
- self.defaults = [ property-set.create
- [ feature.add-defaults $(self.raw) ] ] ;
- }
- return $(self.defaults) ;
- }
-
- rule as-path ( )
- {
- if ! $(self.as-path)
- {
- self.as-path = [ property.as-path [ base ] ] ;
- }
- return $(self.as-path) ;
- }
-
- # Computes the path to be used for a target with the given properties.
- # Returns a list of
- # - the computed path
- # - if the path is relative to the build directory, a value of 'true'.
- #
- rule target-path ( )
- {
- if ! $(self.target-path)
- {
- # The <location> feature can be used to explicitly change the
- # location of generated targets.
- local l = [ get <location> ] ;
- if $(l)
- {
- self.target-path = $(l) ;
- }
- else
- {
- local p = [ property-set.hash-maybe [ as-path ] ] ;
-
- # A real ugly hack. Boost regression test system requires
- # specific target paths, and it seems that changing it to handle
- # other directory layout is really hard. For that reason, we
- # teach V2 to do the things regression system requires. The
- # value of '<location-prefix>' is prepended to the path.
- local prefix = [ get <location-prefix> ] ;
- if $(prefix)
- {
- self.target-path = [ path.join $(prefix) $(p) ] ;
- }
- else
- {
- self.target-path = $(p) ;
- }
- if ! $(self.target-path)
- {
- self.target-path = . ;
- }
- # The path is relative to build dir.
- self.target-path += true ;
- }
- }
- return $(self.target-path) ;
- }
-
- rule add ( ps )
- {
- if ! $(self.added.$(ps))
- {
- self.added.$(ps) = [ property-set.create $(self.raw) [ $(ps).raw ] ]
- ;
- }
- return $(self.added.$(ps)) ;
- }
-
- rule add-raw ( properties * )
- {
- return [ add [ property-set.create $(properties) ] ] ;
- }
-
- # Returns all values of 'feature'.
- #
- rule get ( feature )
- {
- if ! $(self.map-built)
- {
- # For each feature, create a member var and assign all values to it.
- # Since all regular member vars start with 'self', there will be no
- # conflicts between names.
- self.map-built = true ;
- for local v in $(self.raw)
- {
- $(v:G) += $(v:G=) ;
- }
- }
- return $($(feature)) ;
- }
-
- # Returns true if the property-set contains all the
- # specified properties.
- #
- rule contains-raw ( properties * )
- {
- if $(properties) in $(self.raw)
- {
- return true ;
- }
- }
-
- # Returns true if the property-set has values for
- # all the specified features
- #
- rule contains-features ( features * )
- {
- if $(features) in $(self.raw:G)
- {
- return true ;
- }
- }
-
- # private
-
- rule init-base ( )
- {
- for local p in $(self.raw)
- {
- local att = [ feature.attributes $(p:G) ] ;
- # A feature can be both incidental and free, in which case we add it
- # to incidental.
- if incidental in $(att)
- {
- self.incidental += $(p) ;
- }
- else if free in $(att)
- {
- self.free += $(p) ;
- }
- else
- {
- self.base += $(p) ;
- }
- }
- self.base-initialized = true ;
- }
-
- rule init-dependency ( )
- {
- for local p in $(self.raw)
- {
- if dependency in [ feature.attributes $(p:G) ]
- {
- self.dependency += $(p) ;
- }
- else
- {
- self.non-dependency += $(p) ;
- }
- }
- self.dependency-initialized = true ;
- }
-
- rule init-conditional ( )
- {
- for local p in $(self.raw)
- {
- # TODO: Note that non-conditional properties may contain colon (':')
- # characters as well, e.g. free or indirect properties. Indirect
- # properties for example contain a full Jamfile path in their value
- # which on Windows file systems contains ':' as the drive separator.
- if [ MATCH (:) : $(p:G=) ]
- {
- self.conditional += $(p) ;
- }
- else
- {
- self.non-conditional += $(p) ;
- }
- }
- self.conditional-initialized = true ;
- }
-}
-
-
-# Creates a new 'property-set' instance for the given raw properties or returns
-# an already existing ones.
-#
-rule create ( raw-properties * )
-{
- raw-properties = [ sequence.unique
- [ sequence.insertion-sort $(raw-properties) ] ] ;
-
- local key = $(raw-properties:J=-:E=) ;
-
- if ! $(.ps.$(key))
- {
- .ps.$(key) = [ new property-set $(raw-properties) ] ;
- }
- return $(.ps.$(key)) ;
-}
-NATIVE_RULE property-set : create ;
-
-if [ HAS_NATIVE_RULE class@property-set : get : 1 ]
-{
- NATIVE_RULE class@property-set : get ;
-}
-
-if [ HAS_NATIVE_RULE class@property-set : contains-features : 1 ]
-{
- NATIVE_RULE class@property-set : contains-features ;
-}
-
-# Creates a new 'property-set' instance after checking that all properties are
-# valid and converting implicit properties into gristed form.
-#
-rule create-with-validation ( raw-properties * )
-{
- property.validate $(raw-properties) ;
- return [ create [ property.make $(raw-properties) ] ] ;
-}
-
-
-# Creates a property-set from the input given by the user, in the context of
-# 'jamfile-module' at 'location'.
-#
-rule create-from-user-input ( raw-properties * : jamfile-module location )
-{
- local project-id = [ project.attribute $(jamfile-module) id ] ;
- project-id ?= [ path.root $(location) [ path.pwd ] ] ;
- return [ property-set.create [ property.translate $(raw-properties)
- : $(project-id) : $(location) : $(jamfile-module) ] ] ;
-}
-
-
-# Refines requirements with requirements provided by the user. Specially handles
-# "-<property>value" syntax in specification to remove given requirements.
-# - parent-requirements -- property-set object with requirements to refine.
-# - specification -- string list of requirements provided by the user.
-# - project-module -- module to which context indirect features will be
-# bound.
-# - location -- path to which path features are relative.
-#
-rule refine-from-user-input ( parent-requirements : specification * :
- project-module : location )
-{
- if ! $(specification)
- {
- return $(parent-requirements) ;
- }
- else
- {
- local add-requirements ;
- local remove-requirements ;
-
- for local r in $(specification)
- {
- local m = [ MATCH "^-(.*)" : $(r) ] ;
- if $(m)
- {
- remove-requirements += $(m) ;
- }
- else
- {
- add-requirements += $(r) ;
- }
- }
-
- if $(remove-requirements)
- {
- # Need to create a property set, so that path features and indirect
- # features are translated just like they are in project
- # requirements.
- local ps = [ property-set.create-from-user-input
- $(remove-requirements) : $(project-module) $(location) ] ;
-
- parent-requirements = [ property-set.create
- [ set.difference [ $(parent-requirements).raw ]
- : [ $(ps).raw ] ] ] ;
- specification = $(add-requirements) ;
- }
-
- local requirements = [ property-set.create-from-user-input
- $(specification) : $(project-module) $(location) ] ;
-
- return [ $(parent-requirements).refine $(requirements) ] ;
- }
-}
-
-
-# Returns a property-set with an empty set of properties.
-#
-rule empty ( )
-{
- if ! $(.empty)
- {
- .empty = [ create ] ;
- }
- return $(.empty) ;
-}
-
-
-if [ option.get hash : : yes ] = yes
-{
- rule hash-maybe ( path ? )
- {
- path ?= "" ;
- return [ MD5 $(path) ] ;
- }
-}
-else
-{
- rule hash-maybe ( path ? )
- {
- return $(path) ;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/build/property.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/build/property.jam b/ext/kenlm/jam-files/boost-build/build/property.jam
deleted file mode 100644
index dc9dbd8..0000000
--- a/ext/kenlm/jam-files/boost-build/build/property.jam
+++ /dev/null
@@ -1,905 +0,0 @@
-# Copyright 2001, 2002, 2003 Dave Abrahams
-# Copyright 2006 Rene Rivera
-# Copyright 2002, 2003, 2004, 2005, 2006 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-import feature ;
-import indirect ;
-import path ;
-import regex ;
-import string ;
-import sequence ;
-import set ;
-import utility ;
-
-
-# Refines 'properties' by overriding any non-free and non-conditional properties
-# for which a different value is specified in 'requirements'. Returns the
-# resulting list of properties.
-#
-rule refine ( properties * : requirements * )
-{
- local result ;
- local unset ;
-
- # Collect all non-free features in requirements
- for local r in $(requirements)
- {
- # Do not consider conditional requirements.
- if ! [ MATCH (:) : $(r:G=) ] && ! free in [ feature.attributes $(r:G) ]
- {
- unset += $(r:G) ;
- }
- }
-
- # Remove properties that are overridden by requirements
- for local p in $(properties)
- {
- if [ MATCH (:) : $(p:G=) ] || ! $(p:G) in $(unset)
- {
- result += $(p) ;
- }
- }
-
- return [ sequence.unique $(result) $(requirements) ] ;
-}
-
-
-# Removes all conditional properties whose conditions are not met. For those
-# with met conditions, removes the condition. Properties in conditions are
-# looked up in 'context'.
-#
-rule evaluate-conditionals-in-context ( properties * : context * )
-{
- local base ;
- local conditionals ;
- for local p in $(properties)
- {
- if [ MATCH (:<) : $(p) ]
- {
- conditionals += $(p) ;
- }
- else
- {
- base += $(p) ;
- }
- }
-
- local result = $(base) ;
- for local p in $(conditionals)
- {
- # Separate condition and property.
- local s = [ MATCH ^(.*):(<.*) : $(p) ] ;
- # Split condition into individual properties.
- local condition = [ regex.split $(s[1]) "," ] ;
- # Evaluate condition.
- if ! [ MATCH ^(!).* : $(condition:G=) ]
- {
- # Only positive checks
- if $(condition) in $(context)
- {
- result += $(s[2]) ;
- }
- }
- else
- {
- # Have negative checks
- local fail ;
- while $(condition)
- {
- local c = $(condition[1]) ;
- local m = [ MATCH ^!(.*) : $(c) ] ;
- if $(m)
- {
- local p = $(m:G=$(c:G)) ;
- if $(p) in $(context)
- {
- fail = true ;
- c = ;
- }
- }
- else
- {
- if ! $(c) in $(context)
- {
- fail = true ;
- c = ;
- }
- }
- condition = $(condition[2-]) ;
- }
- if ! $(fail)
- {
- result += $(s[2]) ;
- }
- }
- }
- return $(result) ;
-}
-
-
-rule expand-subfeatures-in-conditions ( properties * )
-{
- local result ;
- for local p in $(properties)
- {
- local s = [ MATCH ^(.*):(<.*) : $(p) ] ;
- if ! $(s)
- {
- result += $(p) ;
- }
- else
- {
- local condition = $(s[1]) ;
- local value = $(s[2]) ;
- # Condition might include several elements.
- condition = [ regex.split $(condition) "," ] ;
- local e ;
- for local c in $(condition)
- {
- # It is common for a condition to include a toolset or
- # subfeatures that have not been defined. In that case we want
- # the condition to simply 'never be satisfied' and validation
- # would only produce a spurious error so we prevent it by
- # passing 'true' as the second parameter.
- e += [ feature.expand-subfeatures $(c) : true ] ;
- }
- if $(e) = $(condition)
- {
- # (todo)
- # This is just an optimization and possibly a premature one at
- # that.
- # (todo) (12.07.2008.) (Jurko)
- result += $(p) ;
- }
- else
- {
- result += $(e:J=,):$(value) ;
- }
- }
- }
- return $(result) ;
-}
-
-
-# Helper for as-path, below. Orders properties with the implicit ones first, and
-# within the two sections in alphabetical order of feature name.
-#
-local rule path-order ( x y )
-{
- if $(y:G) && ! $(x:G)
- {
- return true ;
- }
- else if $(x:G) && ! $(y:G)
- {
- return ;
- }
- else
- {
- if ! $(x:G)
- {
- x = [ feature.expand-subfeatures $(x) ] ;
- y = [ feature.expand-subfeatures $(y) ] ;
- }
-
- if $(x[1]) < $(y[1])
- {
- return true ;
- }
- }
-}
-
-
-local rule abbreviate-dashed ( string )
-{
- local r ;
- for local part in [ regex.split $(string) - ]
- {
- r += [ string.abbreviate $(part) ] ;
- }
- return $(r:J=-) ;
-}
-
-
-local rule identity ( string )
-{
- return $(string) ;
-}
-
-
-if --abbreviate-paths in [ modules.peek : ARGV ]
-{
- .abbrev = abbreviate-dashed ;
-}
-else
-{
- .abbrev = identity ;
-}
-
-
-# Returns a path representing the given expanded property set.
-#
-rule as-path ( properties * )
-{
- local entry = .result.$(properties:J=-) ;
-
- if ! $($(entry))
- {
- # Trim redundancy.
- properties = [ feature.minimize $(properties) ] ;
-
- # Sort according to path-order.
- properties = [ sequence.insertion-sort $(properties) : path-order ] ;
-
- local components ;
- for local p in $(properties)
- {
- if $(p:G)
- {
- local f = [ utility.ungrist $(p:G) ] ;
- p = $(f)-$(p:G=) ;
- }
- components += [ $(.abbrev) $(p) ] ;
- }
-
- $(entry) = $(components:J=/) ;
- }
-
- return $($(entry)) ;
-}
-
-
-# Exit with error if property is not valid.
-#
-local rule validate1 ( property )
-{
- local msg ;
- if $(property:G)
- {
- local feature = $(property:G) ;
- local value = $(property:G=) ;
-
- if ! [ feature.valid $(feature) ]
- {
- # Ungrist for better error messages.
- feature = [ utility.ungrist $(property:G) ] ;
- msg = "unknown feature '$(feature)'" ;
- }
- else if $(value) && ! free in [ feature.attributes $(feature) ]
- {
- feature.validate-value-string $(feature) $(value) ;
- }
- else if ! ( $(value) || ( optional in [ feature.attributes $(feature) ] ) )
- {
- # Ungrist for better error messages.
- feature = [ utility.ungrist $(property:G) ] ;
- msg = "No value specified for feature '$(feature)'" ;
- }
- }
- else
- {
- local feature = [ feature.implied-feature $(property) ] ;
- feature.validate-value-string $(feature) $(property) ;
- }
- if $(msg)
- {
- import errors ;
- errors.error "Invalid property "'$(property:J=" ")'": "$(msg:J=" "). ;
- }
-}
-
-
-rule validate ( properties * )
-{
- for local p in $(properties)
- {
- validate1 $(p) ;
- }
-}
-
-
-rule validate-property-sets ( property-sets * )
-{
- for local s in $(property-sets)
- {
- validate [ feature.split $(s) ] ;
- }
-}
-
-
-# Expands any implicit property values in the given property 'specification' so
-# they explicitly state their feature.
-#
-rule make ( specification * )
-{
- local result ;
- for local e in $(specification)
- {
- if $(e:G)
- {
- result += $(e) ;
- }
- else if [ feature.is-implicit-value $(e) ]
- {
- local feature = [ feature.implied-feature $(e) ] ;
- result += $(feature)$(e) ;
- }
- else
- {
- import errors ;
- errors.error "'$(e)' is not a valid property specification" ;
- }
- }
- return $(result) ;
-}
-
-
-# Returns a property set containing all the elements in 'properties' that do not
-# have their attributes listed in 'attributes'.
-#
-rule remove ( attributes + : properties * )
-{
- local result ;
- for local e in $(properties)
- {
- if ! [ set.intersection $(attributes) : [ feature.attributes $(e:G) ] ]
- {
- result += $(e) ;
- }
- }
- return $(result) ;
-}
-
-
-# Returns a property set containing all the elements in 'properties' that have
-# their attributes listed in 'attributes'.
-#
-rule take ( attributes + : properties * )
-{
- local result ;
- for local e in $(properties)
- {
- if [ set.intersection $(attributes) : [ feature.attributes $(e:G) ] ]
- {
- result += $(e) ;
- }
- }
- return $(result) ;
-}
-
-
-# Selects properties corresponding to any of the given features.
-#
-rule select ( features * : properties * )
-{
- local result ;
-
- # Add any missing angle brackets.
- local empty = "" ;
- features = $(empty:G=$(features)) ;
-
- for local p in $(properties)
- {
- if $(p:G) in $(features)
- {
- result += $(p) ;
- }
- }
- return $(result) ;
-}
-
-
-# Returns a modified version of properties with all values of the given feature
-# replaced by the given value. If 'value' is empty the feature will be removed.
-#
-rule change ( properties * : feature value ? )
-{
- local result ;
- for local p in $(properties)
- {
- if $(p:G) = $(feature)
- {
- result += $(value:G=$(feature)) ;
- }
- else
- {
- result += $(p) ;
- }
- }
- return $(result) ;
-}
-
-
-# If 'property' is a conditional property, returns the condition and the
-# property. E.g. <variant>debug,<toolset>gcc:<inlining>full will become
-# <variant>debug,<toolset>gcc <inlining>full. Otherwise, returns an empty
-# string.
-#
-rule split-conditional ( property )
-{
- return [ MATCH "^(.+):(<.+)" : $(property) ] ;
-}
-
-
-rule translate-path-value ( value : path )
-{
- local t ;
- for local v in [ regex.split $(value) "&&" ]
- {
- t += [ path.root [ path.make $(v) ] $(path) ] ;
- }
- return $(t:TJ="&&") ;
-}
-
-rule translate-dependency-value ( value : project-id : project-location )
-{
- local split-target = [ regex.match ^(.*)//(.*) : $(value) ] ;
- if $(split-target)
- {
- local rooted = [ path.root [ path.make $(split-target[1]) ]
- [ path.root $(project-location) [ path.pwd ] ] ] ;
- return $(rooted)//$(split-target[2]) ;
- }
- else if [ path.is-rooted $(value) ]
- {
- return $(value) ;
- }
- else
- {
- return $(project-id)//$(value) ;
- }
-}
-
-rule translate-indirect-value ( rulename : context-module )
-{
- if [ MATCH "^([^%]*)%([^%]+)$" : $(rulename) ]
- {
- # Rule is already in the 'indirect-rule' format.
- return @$(rulename) ;
- }
- else
- {
- local v ;
- if ! [ MATCH "([.])" : $(rulename) ]
- {
- # This is an unqualified rule name. The user might want to
- # set flags on this rule name and toolset.flag
- # auto-qualifies it. Need to do the same here so flag
- # setting works. We can arrange for toolset.flag to *not*
- # auto-qualify the argument but then two rules defined in
- # two Jamfiles would conflict.
- rulename = $(context-module).$(rulename) ;
- }
- v = [ indirect.make $(rulename) : $(context-module) ] ;
- return @$(v) ;
- }
-
-}
-
-# Equivalent to a calling all of:
-# translate-path
-# translate-indirect
-# translate-dependency
-# expand-subfeatures-in-conditions
-# make
-#
-rule translate ( properties * : project-id : project-location : context-module )
-{
- local result ;
- for local p in $(properties)
- {
- local split = [ split-conditional $(p) ] ;
- local condition property ;
-
- if $(split)
- {
- condition = $(split[1]) ;
- property = $(split[2]) ;
-
- local e ;
- for local c in [ regex.split $(condition) "," ]
- {
- e += [ feature.expand-subfeatures $(c) : true ] ;
- }
-
- condition = $(e:J=,): ;
- }
- else
- {
- property = $(p) ;
- }
-
- local feature = $(property:G) ;
- if ! $(feature)
- {
- if [ feature.is-implicit-value $(property) ]
- {
- feature = [ feature.implied-feature $(property) ] ;
- result += $(condition:E=)$(feature)$(property) ;
- }
- else
- {
- import errors ;
- errors.error "'$(e)' is not a valid property specification" ;
- }
- } else {
- local attributes = [ feature.attributes $(feature) ] ;
- local value ;
- # Only free features should be translated
- if free in $(attributes)
- {
- if path in $(attributes)
- {
- value = [ translate-path-value $(property:G=) : $(project-location) ] ;
- result += $(condition:E=)$(feature)$(value) ;
- }
- else if dependency in $(attributes)
- {
- value = [ translate-dependency-value $(property:G=) : $(project-id) : $(project-location) ] ;
- result += $(condition:E=)$(feature)$(value) ;
- }
- else
- {
- local m = [ MATCH ^@(.+) : $(property:G=) ] ;
- if $(m)
- {
- value = [ translate-indirect-value $(m) : $(context-module) ] ;
- result += $(condition:E=)$(feature)$(value) ;
- }
- else
- {
- result += $(condition:E=)$(property) ;
- }
- }
- }
- else
- {
- result += $(condition:E=)$(property) ;
- }
- }
- }
- return $(result) ;
-}
-
-# Interpret all path properties in 'properties' as relative to 'path'. The
-# property values are assumed to be in system-specific form, and will be
-# translated into normalized form.
-#
-rule translate-paths ( properties * : path )
-{
- local result ;
- for local p in $(properties)
- {
- local split = [ split-conditional $(p) ] ;
- local condition = "" ;
- if $(split)
- {
- condition = $(split[1]): ;
- p = $(split[2]) ;
- }
-
- if path in [ feature.attributes $(p:G) ]
- {
- local values = [ regex.split $(p:TG=) "&&" ] ;
- local t ;
- for local v in $(values)
- {
- t += [ path.root [ path.make $(v) ] $(path) ] ;
- }
- t = $(t:J="&&") ;
- result += $(condition)$(t:TG=$(p:G)) ;
- }
- else
- {
- result += $(condition)$(p) ;
- }
- }
- return $(result) ;
-}
-
-
-# Assumes that all feature values that start with '@' are names of rules, used
-# in 'context-module'. Such rules can be either local to the module or global.
-# Converts such values into 'indirect-rule' format (see indirect.jam), so they
-# can be called from other modules. Does nothing for such values that are
-# already in the 'indirect-rule' format.
-#
-rule translate-indirect ( specification * : context-module )
-{
- local result ;
- for local p in $(specification)
- {
- local m = [ MATCH ^@(.+) : $(p:G=) ] ;
- if $(m)
- {
- local v ;
- if [ MATCH "^([^%]*)%([^%]+)$" : $(m) ]
- {
- # Rule is already in the 'indirect-rule' format.
- v = $(m) ;
- }
- else
- {
- if ! [ MATCH "([.])" : $(m) ]
- {
- # This is an unqualified rule name. The user might want to
- # set flags on this rule name and toolset.flag
- # auto-qualifies it. Need to do the same here so flag
- # setting works. We can arrange for toolset.flag to *not*
- # auto-qualify the argument but then two rules defined in
- # two Jamfiles would conflict.
- m = $(context-module).$(m) ;
- }
- v = [ indirect.make $(m) : $(context-module) ] ;
- }
-
- v = @$(v) ;
- result += $(v:G=$(p:G)) ;
- }
- else
- {
- result += $(p) ;
- }
- }
- return $(result) ;
-}
-
-
-# Binds all dependency properties in a list relative to the given project.
-# Targets with absolute paths will be left unchanged and targets which have a
-# project specified will have the path to the project interpreted relative to
-# the specified location.
-#
-rule translate-dependencies ( specification * : project-id : location )
-{
- local result ;
- for local p in $(specification)
- {
- local split = [ split-conditional $(p) ] ;
- local condition = "" ;
- if $(split)
- {
- condition = $(split[1]): ;
- p = $(split[2]) ;
- }
- if dependency in [ feature.attributes $(p:G) ]
- {
- local split-target = [ regex.match ^(.*)//(.*) : $(p:G=) ] ;
- if $(split-target)
- {
- local rooted = [ path.root [ path.make $(split-target[1]) ]
- [ path.root $(location) [ path.pwd ] ] ] ;
- result += $(condition)$(p:G)$(rooted)//$(split-target[2]) ;
- }
- else if [ path.is-rooted $(p:G=) ]
- {
- result += $(condition)$(p) ;
- }
- else
- {
- result += $(condition)$(p:G)$(project-id)//$(p:G=) ;
- }
- }
- else
- {
- result += $(condition)$(p) ;
- }
- }
- return $(result) ;
-}
-
-
-# Class maintaining a property set -> string mapping.
-#
-class property-map
-{
- import numbers ;
- import sequence ;
-
- rule __init__ ( )
- {
- self.next-flag = 1 ;
- }
-
- # Associate 'value' with 'properties'.
- #
- rule insert ( properties * : value )
- {
- self.all-flags += self.$(self.next-flag) ;
- self.$(self.next-flag) = $(value) $(properties) ;
-
- self.next-flag = [ numbers.increment $(self.next-flag) ] ;
- }
-
- # Returns the value associated with 'properties' or any subset of it. If
- # more than one subset has a value assigned to it, returns the value for the
- # longest subset, if it is unique.
- #
- rule find ( property-set )
- {
- # First find all matches.
- local matches ;
- local match-ranks ;
- for local i in $(self.all-flags)
- {
- local list = $($(i)) ;
- if [ $(property-set).contains-raw $(list[2-]) ]
- {
- matches += $(list[1]) ;
- match-ranks += [ sequence.length $(list) ] ;
- }
- }
- local best = [ sequence.select-highest-ranked $(matches)
- : $(match-ranks) ] ;
- if $(best[2])
- {
- import errors : error : errors.error ;
- errors.error "Ambiguous key $(properties:J= :E=)" ;
- }
- return $(best) ;
- }
-
- # Returns the value associated with 'properties'. If 'value' parameter is
- # given, replaces the found value.
- #
- rule find-replace ( properties * : value ? )
- {
- # First find all matches.
- local matches ;
- local match-ranks ;
- for local i in $(self.all-flags)
- {
- if $($(i)[2-]) in $(properties)
- {
- matches += $(i) ;
- match-ranks += [ sequence.length $($(i)) ] ;
- }
- }
- local best = [ sequence.select-highest-ranked $(matches)
- : $(match-ranks) ] ;
- if $(best[2])
- {
- import errors : error : errors.error ;
- errors.error "Ambiguous key $(properties:J= :E=)" ;
- }
- local original = $($(best)[1]) ;
- if $(value)
- {
- $(best) = $(value) $($(best)[2-]) ;
- }
- return $(original) ;
- }
-}
-
-
-rule __test__ ( )
-{
- import assert ;
- import "class" : new ;
- import errors : try catch ;
- import feature ;
-
- # Local rules must be explicitly re-imported.
- import property : path-order abbreviate-dashed ;
-
- feature.prepare-test property-test-temp ;
-
- feature.feature toolset : gcc : implicit symmetric ;
- feature.subfeature toolset gcc : version : 2.95.2 2.95.3 2.95.4 3.0 3.0.1
- 3.0.2 : optional ;
- feature.feature define : : free ;
- feature.feature runtime-link : dynamic static : symmetric link-incompatible ;
- feature.feature optimization : on off ;
- feature.feature variant : debug release : implicit composite symmetric ;
- feature.feature rtti : on off : link-incompatible ;
-
- feature.compose <variant>debug : <define>_DEBUG <optimization>off ;
- feature.compose <variant>release : <define>NDEBUG <optimization>on ;
-
- validate <toolset>gcc <toolset>gcc-3.0.1 : $(test-space) ;
-
- assert.true path-order $(test-space) debug <define>foo ;
- assert.false path-order $(test-space) <define>foo debug ;
- assert.true path-order $(test-space) gcc debug ;
- assert.false path-order $(test-space) debug gcc ;
- assert.true path-order $(test-space) <optimization>on <rtti>on ;
- assert.false path-order $(test-space) <rtti>on <optimization>on ;
-
- assert.result-set-equal <toolset>gcc <rtti>off <define>FOO
- : refine <toolset>gcc <rtti>off
- : <define>FOO
- : $(test-space) ;
-
- assert.result-set-equal <toolset>gcc <optimization>on
- : refine <toolset>gcc <optimization>off
- : <optimization>on
- : $(test-space) ;
-
- assert.result-set-equal <toolset>gcc <rtti>off
- : refine <toolset>gcc : <rtti>off : $(test-space) ;
-
- assert.result-set-equal <toolset>gcc <rtti>off <rtti>off:<define>FOO
- : refine <toolset>gcc : <rtti>off <rtti>off:<define>FOO
- : $(test-space) ;
-
- assert.result-set-equal <toolset>gcc:<define>foo <toolset>gcc:<define>bar
- : refine <toolset>gcc:<define>foo : <toolset>gcc:<define>bar
- : $(test-space) ;
-
- assert.result <define>MY_RELEASE
- : evaluate-conditionals-in-context
- <variant>release,<rtti>off:<define>MY_RELEASE
- : <toolset>gcc <variant>release <rtti>off ;
-
- assert.result debug
- : as-path <optimization>off <variant>debug
- : $(test-space) ;
-
- assert.result gcc/debug/rtti-off
- : as-path <toolset>gcc <optimization>off <rtti>off <variant>debug
- : $(test-space) ;
-
- assert.result optmz-off : abbreviate-dashed optimization-off ;
- assert.result rntm-lnk-sttc : abbreviate-dashed runtime-link-static ;
-
- try ;
- validate <feature>value : $(test-space) ;
- catch "Invalid property '<feature>value': unknown feature 'feature'." ;
-
- try ;
- validate <rtti>default : $(test-space) ;
- catch \"default\" is not a known value of feature <rtti> ;
-
- validate <define>WHATEVER : $(test-space) ;
-
- try ;
- validate <rtti> : $(test-space) ;
- catch "Invalid property '<rtti>': No value specified for feature 'rtti'." ;
-
- try ;
- validate value : $(test-space) ;
- catch \"value\" is not an implicit feature value ;
-
- assert.result-set-equal <rtti>on
- : remove free implicit : <toolset>gcc <define>foo <rtti>on : $(test-space) ;
-
- assert.result-set-equal <include>a
- : select include : <include>a <toolset>gcc ;
-
- assert.result-set-equal <include>a
- : select include bar : <include>a <toolset>gcc ;
-
- assert.result-set-equal <include>a <toolset>gcc
- : select include <bar> <toolset> : <include>a <toolset>gcc ;
-
- assert.result-set-equal <toolset>kylix <include>a
- : change <toolset>gcc <include>a : <toolset> kylix ;
-
- pm = [ new property-map ] ;
- $(pm).insert <toolset>gcc : o ;
- $(pm).insert <toolset>gcc <os>NT : obj ;
- $(pm).insert <toolset>gcc <os>CYGWIN : obj ;
-
- assert.equal o : [ $(pm).find-replace <toolset>gcc ] ;
-
- assert.equal obj : [ $(pm).find-replace <toolset>gcc <os>NT ] ;
-
- try ;
- $(pm).find-replace <toolset>gcc <os>NT <os>CYGWIN ;
- catch "Ambiguous key <toolset>gcc <os>NT <os>CYGWIN" ;
-
- # Test ordinary properties.
- assert.result : split-conditional <toolset>gcc ;
-
- # Test properties with ":".
- assert.result : split-conditional <define>FOO=A::B ;
-
- # Test conditional feature.
- assert.result-set-equal <toolset>gcc,<toolset-gcc:version>3.0 <define>FOO
- : split-conditional <toolset>gcc,<toolset-gcc:version>3.0:<define>FOO ;
-
- feature.finish-test property-test-temp ;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/build/readme.txt
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/build/readme.txt b/ext/kenlm/jam-files/boost-build/build/readme.txt
deleted file mode 100644
index b15055b..0000000
--- a/ext/kenlm/jam-files/boost-build/build/readme.txt
+++ /dev/null
@@ -1,11 +0,0 @@
-Copyright 2001, 2002 Dave Abrahams
-Copyright 2002 Vladimir Prus
-Distributed under the Boost Software License, Version 1.0.
-(See accompanying file LICENSE_1_0.txt or copy at
-http://www.boost.org/LICENSE_1_0.txt)
-
-Development code for new build system. To run unit tests for jam code, execute:
-
- bjam --debug --build-system=test
-
-Comprehensive tests require Python. See ../test/readme.txt
[24/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/function.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/function.c b/ext/kenlm/jam-files/engine/function.c
deleted file mode 100644
index 690855e..0000000
--- a/ext/kenlm/jam-files/engine/function.c
+++ /dev/null
@@ -1,4870 +0,0 @@
-/*
- * Copyright 2011 Steven Watanabe
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#include "jam.h"
-#include "function.h"
-
-#include "class.h"
-#include "compile.h"
-#include "constants.h"
-#include "filesys.h"
-#include "frames.h"
-#include "lists.h"
-#include "mem.h"
-#include "pathsys.h"
-#include "rules.h"
-#include "search.h"
-#include "variable.h"
-
-#include <assert.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#ifdef OS_CYGWIN
-# include <cygwin/version.h>
-# include <sys/cygwin.h>
-# ifdef CYGWIN_VERSION_CYGWIN_CONV
-# include <errno.h>
-# endif
-# include <windows.h>
-#endif
-
-int glob( char const * s, char const * c );
-void backtrace( FRAME * );
-void backtrace_line( FRAME * );
-
-#define INSTR_PUSH_EMPTY 0
-#define INSTR_PUSH_CONSTANT 1
-#define INSTR_PUSH_ARG 2
-#define INSTR_PUSH_VAR 3
-#define INSTR_PUSH_VAR_FIXED 57
-#define INSTR_PUSH_GROUP 4
-#define INSTR_PUSH_RESULT 5
-#define INSTR_PUSH_APPEND 6
-#define INSTR_SWAP 7
-
-#define INSTR_JUMP_EMPTY 8
-#define INSTR_JUMP_NOT_EMPTY 9
-
-#define INSTR_JUMP 10
-#define INSTR_JUMP_LT 11
-#define INSTR_JUMP_LE 12
-#define INSTR_JUMP_GT 13
-#define INSTR_JUMP_GE 14
-#define INSTR_JUMP_EQ 15
-#define INSTR_JUMP_NE 16
-#define INSTR_JUMP_IN 17
-#define INSTR_JUMP_NOT_IN 18
-
-#define INSTR_JUMP_NOT_GLOB 19
-
-#define INSTR_FOR_INIT 56
-#define INSTR_FOR_LOOP 20
-
-#define INSTR_SET_RESULT 21
-#define INSTR_RETURN 22
-#define INSTR_POP 23
-
-#define INSTR_PUSH_LOCAL 24
-#define INSTR_POP_LOCAL 25
-#define INSTR_SET 26
-#define INSTR_APPEND 27
-#define INSTR_DEFAULT 28
-
-#define INSTR_PUSH_LOCAL_FIXED 58
-#define INSTR_POP_LOCAL_FIXED 59
-#define INSTR_SET_FIXED 60
-#define INSTR_APPEND_FIXED 61
-#define INSTR_DEFAULT_FIXED 62
-
-#define INSTR_PUSH_LOCAL_GROUP 29
-#define INSTR_POP_LOCAL_GROUP 30
-#define INSTR_SET_GROUP 31
-#define INSTR_APPEND_GROUP 32
-#define INSTR_DEFAULT_GROUP 33
-
-#define INSTR_PUSH_ON 34
-#define INSTR_POP_ON 35
-#define INSTR_SET_ON 36
-#define INSTR_APPEND_ON 37
-#define INSTR_DEFAULT_ON 38
-#define INSTR_GET_ON 65
-
-#define INSTR_CALL_RULE 39
-#define INSTR_CALL_MEMBER_RULE 66
-
-#define INSTR_APPLY_MODIFIERS 40
-#define INSTR_APPLY_INDEX 41
-#define INSTR_APPLY_INDEX_MODIFIERS 42
-#define INSTR_APPLY_MODIFIERS_GROUP 43
-#define INSTR_APPLY_INDEX_GROUP 44
-#define INSTR_APPLY_INDEX_MODIFIERS_GROUP 45
-#define INSTR_COMBINE_STRINGS 46
-#define INSTR_GET_GRIST 64
-
-#define INSTR_INCLUDE 47
-#define INSTR_RULE 48
-#define INSTR_ACTIONS 49
-#define INSTR_PUSH_MODULE 50
-#define INSTR_POP_MODULE 51
-#define INSTR_CLASS 52
-#define INSTR_BIND_MODULE_VARIABLES 63
-
-#define INSTR_APPEND_STRINGS 53
-#define INSTR_WRITE_FILE 54
-#define INSTR_OUTPUT_STRINGS 55
-
-typedef struct instruction
-{
- unsigned int op_code;
- int arg;
-} instruction;
-
-typedef struct _subfunction
-{
- OBJECT * name;
- FUNCTION * code;
- int local;
-} SUBFUNCTION;
-
-typedef struct _subaction
-{
- OBJECT * name;
- FUNCTION * command;
- int flags;
-} SUBACTION;
-
-#define FUNCTION_BUILTIN 0
-#define FUNCTION_JAM 1
-
-struct argument
-{
- int flags;
-#define ARG_ONE 0
-#define ARG_OPTIONAL 1
-#define ARG_PLUS 2
-#define ARG_STAR 3
-#define ARG_VARIADIC 4
- OBJECT * type_name;
- OBJECT * arg_name;
- int index;
-};
-
-struct arg_list
-{
- int size;
- struct argument * args;
-};
-
-struct _function
-{
- int type;
- int reference_count;
- OBJECT * rulename;
- struct arg_list * formal_arguments;
- int num_formal_arguments;
-};
-
-typedef struct _builtin_function
-{
- FUNCTION base;
- LIST * ( * func )( FRAME *, int flags );
- int flags;
-} BUILTIN_FUNCTION;
-
-typedef struct _jam_function
-{
- FUNCTION base;
- int code_size;
- instruction * code;
- int num_constants;
- OBJECT * * constants;
- int num_subfunctions;
- SUBFUNCTION * functions;
- int num_subactions;
- SUBACTION * actions;
- FUNCTION * generic;
- OBJECT * file;
- int line;
-} JAM_FUNCTION;
-
-
-#ifdef HAVE_PYTHON
-
-#define FUNCTION_PYTHON 2
-
-typedef struct _python_function
-{
- FUNCTION base;
- PyObject * python_function;
-} PYTHON_FUNCTION;
-
-static LIST * call_python_function( PYTHON_FUNCTION *, FRAME * );
-
-#endif
-
-
-struct _stack
-{
- void * data;
-};
-
-static void * stack;
-
-STACK * stack_global()
-{
- static STACK result;
- if ( !stack )
- {
- int const size = 1 << 21;
- stack = BJAM_MALLOC( size );
- result.data = (char *)stack + size;
- }
- return &result;
-}
-
-static void check_alignment( STACK * s )
-{
- assert( (size_t)s->data % sizeof( LIST * ) == 0 );
-}
-
-void * stack_allocate( STACK * s, int size )
-{
- check_alignment( s );
- s->data = (char *)s->data - size;
- check_alignment( s );
- return s->data;
-}
-
-void stack_deallocate( STACK * s, int size )
-{
- check_alignment( s );
- s->data = (char *)s->data + size;
- check_alignment( s );
-}
-
-void stack_push( STACK * s, LIST * l )
-{
- *(LIST * *)stack_allocate( s, sizeof( LIST * ) ) = l;
-}
-
-LIST * stack_pop( STACK * s )
-{
- LIST * const result = *(LIST * *)s->data;
- stack_deallocate( s, sizeof( LIST * ) );
- return result;
-}
-
-LIST * stack_top( STACK * s )
-{
- check_alignment( s );
- return *(LIST * *)s->data;
-}
-
-LIST * stack_at( STACK * s, int n )
-{
- check_alignment( s );
- return *( (LIST * *)s->data + n );
-}
-
-void stack_set( STACK * s, int n, LIST * value )
-{
- check_alignment( s );
- *((LIST * *)s->data + n) = value;
-}
-
-void * stack_get( STACK * s )
-{
- check_alignment( s );
- return s->data;
-}
-
-LIST * frame_get_local( FRAME * frame, int idx )
-{
- /* The only local variables are the arguments. */
- return list_copy( lol_get( frame->args, idx ) );
-}
-
-static OBJECT * function_get_constant( JAM_FUNCTION * function, int idx )
-{
- return function->constants[ idx ];
-}
-
-static LIST * function_get_variable( JAM_FUNCTION * function, FRAME * frame,
- int idx )
-{
- return list_copy( var_get( frame->module, function->constants[ idx ] ) );
-}
-
-static void function_set_variable( JAM_FUNCTION * function, FRAME * frame,
- int idx, LIST * value )
-{
- var_set( frame->module, function->constants[ idx ], value, VAR_SET );
-}
-
-static LIST * function_swap_variable( JAM_FUNCTION * function, FRAME * frame,
- int idx, LIST * value )
-{
- return var_swap( frame->module, function->constants[ idx ], value );
-}
-
-static void function_append_variable( JAM_FUNCTION * function, FRAME * frame,
- int idx, LIST * value )
-{
- var_set( frame->module, function->constants[ idx ], value, VAR_APPEND );
-}
-
-static void function_default_variable( JAM_FUNCTION * function, FRAME * frame,
- int idx, LIST * value )
-{
- var_set( frame->module, function->constants[ idx ], value, VAR_DEFAULT );
-}
-
-static void function_set_rule( JAM_FUNCTION * function, FRAME * frame,
- STACK * s, int idx )
-{
- SUBFUNCTION * sub = function->functions + idx;
- new_rule_body( frame->module, sub->name, sub->code, !sub->local );
-}
-
-static void function_set_actions( JAM_FUNCTION * function, FRAME * frame,
- STACK * s, int idx )
-{
- SUBACTION * sub = function->actions + idx;
- LIST * bindlist = stack_pop( s );
- new_rule_actions( frame->module, sub->name, sub->command, bindlist,
- sub->flags );
-}
-
-
-/*
- * Returns the index if name is "<", ">", "1", "2", ... or "19" otherwise
- * returns -1.
- */
-
-static int get_argument_index( char const * s )
-{
- if ( s[ 0 ] != '\0')
- {
- if ( s[ 1 ] == '\0' )
- {
- switch ( s[ 0 ] )
- {
- case '<': return 0;
- case '>': return 1;
-
- case '1':
- case '2':
- case '3':
- case '4':
- case '5':
- case '6':
- case '7':
- case '8':
- case '9':
- return s[ 0 ] - '1';
- }
- }
- else if ( s[ 0 ] == '1' && s[ 2 ] == '\0' )
- {
- switch( s[ 1 ] )
- {
- case '0':
- case '1':
- case '2':
- case '3':
- case '4':
- case '5':
- case '6':
- case '7':
- case '8':
- case '9':
- return s[ 1 ] - '0' + 10 - 1;
- }
- }
- }
- return -1;
-}
-
-static LIST * function_get_named_variable( JAM_FUNCTION * function,
- FRAME * frame, OBJECT * name )
-{
- int const idx = get_argument_index( object_str( name ) );
- return idx == -1
- ? list_copy( var_get( frame->module, name ) )
- : list_copy( lol_get( frame->args, idx ) );
-}
-
-static void function_set_named_variable( JAM_FUNCTION * function, FRAME * frame,
- OBJECT * name, LIST * value)
-{
- var_set( frame->module, name, value, VAR_SET );
-}
-
-static LIST * function_swap_named_variable( JAM_FUNCTION * function,
- FRAME * frame, OBJECT * name, LIST * value )
-{
- return var_swap( frame->module, name, value );
-}
-
-static void function_append_named_variable( JAM_FUNCTION * function,
- FRAME * frame, OBJECT * name, LIST * value)
-{
- var_set( frame->module, name, value, VAR_APPEND );
-}
-
-static void function_default_named_variable( JAM_FUNCTION * function,
- FRAME * frame, OBJECT * name, LIST * value )
-{
- var_set( frame->module, name, value, VAR_DEFAULT );
-}
-
-static LIST * function_call_rule( JAM_FUNCTION * function, FRAME * frame,
- STACK * s, int n_args, char const * unexpanded, OBJECT * file, int line )
-{
- FRAME inner[ 1 ];
- int i;
- LIST * first = stack_pop( s );
- LIST * result = L0;
- OBJECT * rulename;
- LIST * trailing;
-
- frame->file = file;
- frame->line = line;
-
- if ( list_empty( first ) )
- {
- backtrace_line( frame );
- printf( "warning: rulename %s expands to empty string\n", unexpanded );
- backtrace( frame );
- list_free( first );
- for ( i = 0; i < n_args; ++i )
- list_free( stack_pop( s ) );
- return result;
- }
-
- rulename = object_copy( list_front( first ) );
-
- frame_init( inner );
- inner->prev = frame;
- inner->prev_user = frame->module->user_module ? frame : frame->prev_user;
- inner->module = frame->module; /* This gets fixed up in evaluate_rule(). */
-
- for ( i = 0; i < n_args; ++i )
- lol_add( inner->args, stack_at( s, n_args - i - 1 ) );
-
- for ( i = 0; i < n_args; ++i )
- stack_pop( s );
-
- trailing = list_pop_front( first );
- if ( trailing )
- {
- if ( inner->args->count == 0 )
- lol_add( inner->args, trailing );
- else
- {
- LIST * * const l = &inner->args->list[ 0 ];
- *l = list_append( trailing, *l );
- }
- }
-
- result = evaluate_rule( bindrule( rulename, inner->module ), rulename, inner );
- frame_free( inner );
- object_free( rulename );
- return result;
-}
-
-static LIST * function_call_member_rule( JAM_FUNCTION * function, FRAME * frame, STACK * s, int n_args, OBJECT * rulename, OBJECT * file, int line )
-{
- FRAME inner[ 1 ];
- int i;
- LIST * first = stack_pop( s );
- LIST * result = L0;
- LIST * trailing;
- RULE * rule;
- module_t * module;
- OBJECT * real_rulename = 0;
-
- frame->file = file;
- frame->line = line;
-
- if ( list_empty( first ) )
- {
- backtrace_line( frame );
- printf( "warning: object is empty\n" );
- backtrace( frame );
-
- list_free( first );
-
- for( i = 0; i < n_args; ++i )
- {
- list_free( stack_pop( s ) );
- }
-
- return result;
- }
-
- /* FIXME: handle generic case */
- assert( list_length( first ) == 1 );
-
- module = bindmodule( list_front( first ) );
- if ( module->class_module )
- {
- rule = bindrule( rulename, module );
- real_rulename = object_copy( function_rulename( rule->procedure ) );
- }
- else
- {
- string buf[ 1 ];
- string_new( buf );
- string_append( buf, object_str( list_front( first ) ) );
- string_push_back( buf, '.' );
- string_append( buf, object_str( rulename ) );
- real_rulename = object_new( buf->value );
- string_free( buf );
- rule = bindrule( real_rulename, frame->module );
- }
-
- frame_init( inner );
-
- inner->prev = frame;
- inner->prev_user = frame->module->user_module ? frame : frame->prev_user;
- inner->module = frame->module; /* This gets fixed up in evaluate_rule(), below. */
-
- for( i = 0; i < n_args; ++i )
- {
- lol_add( inner->args, stack_at( s, n_args - i - 1 ) );
- }
-
- for( i = 0; i < n_args; ++i )
- {
- stack_pop( s );
- }
-
- if ( list_length( first ) > 1 )
- {
- string buf[ 1 ];
- LIST * trailing = L0;
- LISTITER iter = list_begin( first ), end = list_end( first );
- iter = list_next( iter );
- string_new( buf );
- for ( ; iter != end; iter = list_next( iter ) )
- {
- string_append( buf, object_str( list_item( iter ) ) );
- string_push_back( buf, '.' );
- string_append( buf, object_str( rulename ) );
- trailing = list_push_back( trailing, object_new( buf->value ) );
- string_truncate( buf, 0 );
- }
- string_free( buf );
- if ( inner->args->count == 0 )
- lol_add( inner->args, trailing );
- else
- {
- LIST * * const l = &inner->args->list[ 0 ];
- *l = list_append( trailing, *l );
- }
- }
-
- result = evaluate_rule( rule, real_rulename, inner );
- frame_free( inner );
- object_free( rulename );
- object_free( real_rulename );
- return result;
-}
-
-
-/* Variable expansion */
-
-typedef struct
-{
- int sub1;
- int sub2;
-} subscript_t;
-
-typedef struct
-{
- PATHNAME f; /* :GDBSMR -- pieces */
- char parent; /* :P -- go to parent directory */
- char filemods; /* one of the above applied */
- char downshift; /* :L -- downshift result */
- char upshift; /* :U -- upshift result */
- char to_slashes; /* :T -- convert "\" to "/" */
- char to_windows; /* :W -- convert cygwin to native paths */
- PATHPART empty; /* :E -- default for empties */
- PATHPART join; /* :J -- join list with char */
-} VAR_EDITS;
-
-static LIST * apply_modifiers_impl( LIST * result, string * buf,
- VAR_EDITS * edits, int n, LISTITER iter, LISTITER end );
-static void get_iters( subscript_t const subscript, LISTITER * const first,
- LISTITER * const last, int const length );
-
-
-/*
- * var_edit_parse() - parse : modifiers into PATHNAME structure
- *
- * The : modifiers in a $(varname:modifier) currently support replacing or
- * omitting elements of a filename, and so they are parsed into a PATHNAME
- * structure (which contains pointers into the original string).
- *
- * Modifiers of the form "X=value" replace the component X with the given value.
- * Modifiers without the "=value" cause everything but the component X to be
- * omitted. X is one of:
- *
- * G <grist>
- * D directory name
- * B base name
- * S .suffix
- * M (member)
- * R root directory - prepended to whole path
- *
- * This routine sets:
- *
- * f->f_xxx.ptr = 0
- * f->f_xxx.len = 0
- * -> leave the original component xxx
- *
- * f->f_xxx.ptr = string
- * f->f_xxx.len = strlen( string )
- * -> replace component xxx with string
- *
- * f->f_xxx.ptr = ""
- * f->f_xxx.len = 0
- * -> omit component xxx
- *
- * var_edit_file() below and path_build() obligingly follow this convention.
- */
-
-static int var_edit_parse( char const * mods, VAR_EDITS * edits, int havezeroed
- )
-{
- while ( *mods )
- {
- PATHPART * fp;
-
- switch ( *mods++ )
- {
- case 'L': edits->downshift = 1; continue;
- case 'U': edits->upshift = 1; continue;
- case 'P': edits->parent = edits->filemods = 1; continue;
- case 'E': fp = &edits->empty; goto strval;
- case 'J': fp = &edits->join; goto strval;
- case 'G': fp = &edits->f.f_grist; goto fileval;
- case 'R': fp = &edits->f.f_root; goto fileval;
- case 'D': fp = &edits->f.f_dir; goto fileval;
- case 'B': fp = &edits->f.f_base; goto fileval;
- case 'S': fp = &edits->f.f_suffix; goto fileval;
- case 'M': fp = &edits->f.f_member; goto fileval;
- case 'T': edits->to_slashes = 1; continue;
- case 'W': edits->to_windows = 1; continue;
- default:
- continue; /* Should complain, but so what... */
- }
-
- fileval:
- /* Handle :CHARS, where each char (without a following =) selects a
- * particular file path element. On the first such char, we deselect all
- * others (by setting ptr = "", len = 0) and for each char we select
- * that element (by setting ptr = 0).
- */
- edits->filemods = 1;
-
- if ( *mods != '=' )
- {
- if ( !havezeroed++ )
- {
- int i;
- for ( i = 0; i < 6; ++i )
- {
- edits->f.part[ i ].len = 0;
- edits->f.part[ i ].ptr = "";
- }
- }
-
- fp->ptr = 0;
- continue;
- }
-
- strval:
- /* Handle :X=value, or :X */
- if ( *mods != '=' )
- {
- fp->ptr = "";
- fp->len = 0;
- }
- else
- {
- fp->ptr = ++mods;
- fp->len = strlen( mods );
- mods += fp->len;
- }
- }
-
- return havezeroed;
-}
-
-
-/*
- * var_edit_file() - copy input target name to output, modifying filename.
- */
-
-static void var_edit_file( char const * in, string * out, VAR_EDITS * edits )
-{
- if ( edits->filemods )
- {
- PATHNAME pathname;
-
- /* Parse apart original filename, putting parts into "pathname". */
- path_parse( in, &pathname );
-
- /* Replace any pathname with edits->f */
- if ( edits->f.f_grist .ptr ) pathname.f_grist = edits->f.f_grist;
- if ( edits->f.f_root .ptr ) pathname.f_root = edits->f.f_root;
- if ( edits->f.f_dir .ptr ) pathname.f_dir = edits->f.f_dir;
- if ( edits->f.f_base .ptr ) pathname.f_base = edits->f.f_base;
- if ( edits->f.f_suffix.ptr ) pathname.f_suffix = edits->f.f_suffix;
- if ( edits->f.f_member.ptr ) pathname.f_member = edits->f.f_member;
-
- /* If requested, modify pathname to point to parent. */
- if ( edits->parent )
- path_parent( &pathname );
-
- /* Put filename back together. */
- path_build( &pathname, out );
- }
- else
- string_append( out, in );
-}
-
-/*
- * var_edit_cyg2win() - conversion of a cygwin to a Windows path.
- *
- * FIXME: skip grist
- */
-
-#ifdef OS_CYGWIN
-static void var_edit_cyg2win( string * out, size_t pos, VAR_EDITS * edits )
-{
- if ( edits->to_windows )
- {
- #ifdef CYGWIN_VERSION_CYGWIN_CONV
- /* Use new Cygwin API added with Cygwin 1.7. Old one had no error
- * handling and has been deprecated.
- */
- char * dynamicBuffer = 0;
- char buffer[ MAX_PATH + 1001 ];
- char const * result = buffer;
- cygwin_conv_path_t const conv_type = CCP_POSIX_TO_WIN_A | CCP_RELATIVE;
- ssize_t const apiResult = cygwin_conv_path( conv_type, out->value + pos,
- buffer, sizeof( buffer ) / sizeof( *buffer ) );
- assert( apiResult == 0 || apiResult == -1 );
- assert( apiResult || strlen( result ) < sizeof( buffer ) / sizeof(
- *buffer ) );
- if ( apiResult )
- {
- result = 0;
- if ( errno == ENOSPC )
- {
- ssize_t const size = cygwin_conv_path( conv_type, out->value +
- pos, NULL, 0 );
- assert( size >= -1 );
- if ( size > 0 )
- {
- dynamicBuffer = (char *)BJAM_MALLOC_ATOMIC( size );
- if ( dynamicBuffer )
- {
- ssize_t const apiResult = cygwin_conv_path( conv_type,
- out->value + pos, dynamicBuffer, size );
- assert( apiResult == 0 || apiResult == -1 );
- if ( !apiResult )
- {
- result = dynamicBuffer;
- assert( strlen( result ) < size );
- }
- }
- }
- }
- }
- #else /* CYGWIN_VERSION_CYGWIN_CONV */
- /* Use old Cygwin API deprecated with Cygwin 1.7. */
- char result[ MAX_PATH + 1 ];
- cygwin_conv_to_win32_path( out->value + pos, result );
- assert( strlen( result ) <= MAX_PATH );
- #endif /* CYGWIN_VERSION_CYGWIN_CONV */
- if ( result )
- {
- string_truncate( out, pos );
- string_append( out, result );
- edits->to_slashes = 0;
- }
- #ifdef CYGWIN_VERSION_CYGWIN_CONV
- if ( dynamicBuffer )
- BJAM_FREE( dynamicBuffer );
- #endif
- }
-}
-#endif /* OS_CYGWIN */
-
-
-/*
- * var_edit_shift() - do upshift/downshift & other mods.
- */
-
-static void var_edit_shift( string * out, size_t pos, VAR_EDITS * edits )
-{
-#ifdef OS_CYGWIN
- var_edit_cyg2win( out, pos, edits );
-#endif
-
- if ( edits->upshift || edits->downshift || edits->to_slashes )
- {
- /* Handle upshifting, downshifting and slash translation now. */
- char * p;
- for ( p = out->value + pos; *p; ++p )
- {
- if ( edits->upshift )
- *p = toupper( *p );
- else if ( edits->downshift )
- *p = tolower( *p );
- if ( edits->to_slashes && ( *p == '\\' ) )
- *p = '/';
- }
- }
-}
-
-
-/*
- * Reads n LISTs from the top of the STACK and combines them to form VAR_EDITS.
- * Returns the number of VAR_EDITS pushed onto the STACK.
- */
-
-static int expand_modifiers( STACK * s, int n )
-{
- int i;
- int total = 1;
- LIST * * args = stack_get( s );
- for ( i = 0; i < n; ++i )
- total *= list_length( args[ i ] );
-
- if ( total != 0 )
- {
- VAR_EDITS * out = stack_allocate( s, total * sizeof( VAR_EDITS ) );
- LISTITER * iter = stack_allocate( s, n * sizeof( LIST * ) );
- for ( i = 0; i < n; ++i )
- iter[ i ] = list_begin( args[ i ] );
- i = 0;
- {
- int havezeroed;
- loop:
- memset( out, 0, sizeof( *out ) );
- havezeroed = 0;
- for ( i = 0; i < n; ++i )
- havezeroed = var_edit_parse( object_str( list_item( iter[ i ] )
- ), out, havezeroed );
- ++out;
- while ( --i >= 0 )
- {
- if ( list_next( iter[ i ] ) != list_end( args[ i ] ) )
- {
- iter[ i ] = list_next( iter[ i ] );
- goto loop;
- }
- iter[ i ] = list_begin( args[ i ] );
- }
- }
- stack_deallocate( s, n * sizeof( LIST * ) );
- }
- return total;
-}
-
-static LIST * apply_modifiers( STACK * s, int n )
-{
- LIST * value = stack_top( s );
- LIST * result = L0;
- VAR_EDITS * const edits = (VAR_EDITS *)( (LIST * *)stack_get( s ) + 1 );
- string buf[ 1 ];
- string_new( buf );
- result = apply_modifiers_impl( result, buf, edits, n, list_begin( value ),
- list_end( value ) );
- string_free( buf );
- return result;
-}
-
-
-/*
- * Parse a string of the form "1-2", "-2--1", "2-" and return the two
- * subscripts.
- */
-
-subscript_t parse_subscript( char const * s )
-{
- subscript_t result;
- result.sub1 = 0;
- result.sub2 = 0;
- do /* so we can use "break" */
- {
- /* Allow negative subscripts. */
- if ( !isdigit( *s ) && ( *s != '-' ) )
- {
- result.sub2 = 0;
- break;
- }
- result.sub1 = atoi( s );
-
- /* Skip over the first symbol, which is either a digit or dash. */
- ++s;
- while ( isdigit( *s ) ) ++s;
-
- if ( *s == '\0' )
- {
- result.sub2 = result.sub1;
- break;
- }
-
- if ( *s != '-' )
- {
- result.sub2 = 0;
- break;
- }
-
- ++s;
-
- if ( *s == '\0' )
- {
- result.sub2 = -1;
- break;
- }
-
- if ( !isdigit( *s ) && ( *s != '-' ) )
- {
- result.sub2 = 0;
- break;
- }
-
- /* First, compute the index of the last element. */
- result.sub2 = atoi( s );
- while ( isdigit( *++s ) );
-
- if ( *s != '\0' )
- result.sub2 = 0;
-
- } while ( 0 );
- return result;
-}
-
-static LIST * apply_subscript( STACK * s )
-{
- LIST * value = stack_top( s );
- LIST * indices = stack_at( s, 1 );
- LIST * result = L0;
- int length = list_length( value );
- string buf[ 1 ];
- LISTITER indices_iter = list_begin( indices );
- LISTITER const indices_end = list_end( indices );
- string_new( buf );
- for ( ; indices_iter != indices_end; indices_iter = list_next( indices_iter
- ) )
- {
- LISTITER iter = list_begin( value );
- LISTITER end = list_end( value );
- subscript_t const subscript = parse_subscript( object_str( list_item(
- indices_iter ) ) );
- get_iters( subscript, &iter, &end, length );
- for ( ; iter != end; iter = list_next( iter ) )
- result = list_push_back( result, object_copy( list_item( iter ) ) );
- }
- string_free( buf );
- return result;
-}
-
-
-/*
- * Reads the LIST from first and applies subscript to it. The results are
- * written to *first and *last.
- */
-
-static void get_iters( subscript_t const subscript, LISTITER * const first,
- LISTITER * const last, int const length )
-{
- int start;
- int size;
- LISTITER iter;
- LISTITER end;
- {
-
- if ( subscript.sub1 < 0 )
- start = length + subscript.sub1;
- else if ( subscript.sub1 > length )
- start = length;
- else
- start = subscript.sub1 - 1;
-
- size = subscript.sub2 < 0
- ? length + 1 + subscript.sub2 - start
- : subscript.sub2 - start;
-
- /*
- * HACK: When the first subscript is before the start of the list, it
- * magically becomes the beginning of the list. This is inconsistent,
- * but needed for backwards compatibility.
- */
- if ( start < 0 )
- start = 0;
-
- /* The "sub2 < 0" test handles the semantic error of sub2 < sub1. */
- if ( size < 0 )
- size = 0;
-
- if ( start + size > length )
- size = length - start;
- }
-
- iter = *first;
- while ( start-- > 0 )
- iter = list_next( iter );
-
- end = iter;
- while ( size-- > 0 )
- end = list_next( end );
-
- *first = iter;
- *last = end;
-}
-
-static LIST * apply_modifiers_empty( LIST * result, string * buf,
- VAR_EDITS * edits, int n )
-{
- int i;
- for ( i = 0; i < n; ++i )
- {
- if ( edits[ i ].empty.ptr )
- {
- /** FIXME: is empty.ptr always null-terminated? */
- var_edit_file( edits[ i ].empty.ptr, buf, edits + i );
- var_edit_shift( buf, 0, edits + i );
- result = list_push_back( result, object_new( buf->value ) );
- string_truncate( buf, 0 );
- }
- }
- return result;
-}
-
-static LIST * apply_modifiers_non_empty( LIST * result, string * buf,
- VAR_EDITS * edits, int n, LISTITER begin, LISTITER end )
-{
- int i;
- LISTITER iter;
- for ( i = 0; i < n; ++i )
- {
- if ( edits[ i ].join.ptr )
- {
- var_edit_file( object_str( list_item( begin ) ), buf, edits + i );
- var_edit_shift( buf, 0, edits + i );
- for ( iter = list_next( begin ); iter != end; iter = list_next( iter
- ) )
- {
- size_t size;
- string_append( buf, edits[ i ].join.ptr );
- size = buf->size;
- var_edit_file( object_str( list_item( iter ) ), buf, edits + i
- );
- var_edit_shift( buf, size, edits + i );
- }
- result = list_push_back( result, object_new( buf->value ) );
- string_truncate( buf, 0 );
- }
- else
- {
- for ( iter = begin; iter != end; iter = list_next( iter ) )
- {
- var_edit_file( object_str( list_item( iter ) ), buf, edits + i );
- var_edit_shift( buf, 0, edits + i );
- result = list_push_back( result, object_new( buf->value ) );
- string_truncate( buf, 0 );
- }
- }
- }
- return result;
-}
-
-static LIST * apply_modifiers_impl( LIST * result, string * buf,
- VAR_EDITS * edits, int n, LISTITER iter, LISTITER end )
-{
- return iter == end
- ? apply_modifiers_empty( result, buf, edits, n )
- : apply_modifiers_non_empty( result, buf, edits, n, iter, end );
-}
-
-static LIST * apply_subscript_and_modifiers( STACK * s, int n )
-{
- LIST * const value = stack_top( s );
- LIST * const indices = stack_at( s, 1 );
- LIST * result = L0;
- VAR_EDITS * const edits = (VAR_EDITS *)((LIST * *)stack_get( s ) + 2);
- int const length = list_length( value );
- string buf[ 1 ];
- LISTITER indices_iter = list_begin( indices );
- LISTITER const indices_end = list_end( indices );
- string_new( buf );
- for ( ; indices_iter != indices_end; indices_iter = list_next( indices_iter
- ) )
- {
- LISTITER iter = list_begin( value );
- LISTITER end = list_end( value );
- subscript_t const sub = parse_subscript( object_str( list_item(
- indices_iter ) ) );
- get_iters( sub, &iter, &end, length );
- result = apply_modifiers_impl( result, buf, edits, n, iter, end );
- }
- string_free( buf );
- return result;
-}
-
-
-/*
- * expand() - expands a list of concatenated strings and variable refereces
- *
- * Takes a list of expansion items - each representing one element to be
- * concatenated and each containing a list of its values. Returns a list of all
- * possible values constructed by selecting a single value from each of the
- * elements and concatenating them together.
- *
- * For example, in the following code:
- *
- * local a = one two three four ;
- * local b = foo bar ;
- * ECHO /$(a)/$(b)/$(a)/ ;
- *
- * When constructing the result of /$(a)/$(b)/ this function would get called
- * with the following 7 expansion items:
- * 1. /
- * 2. one two three four
- * 3. /
- * 4. foo bar
- * 5. /
- * 6. one two three four
- * 7. /
- *
- * And would result in a list containing 32 values:
- * 1. /one/foo/one/
- * 2. /one/foo/two/
- * 3. /one/foo/three/
- * 4. /one/foo/four/
- * 5. /one/bar/one/
- * ...
- *
- */
-
-typedef struct expansion_item
-{
- /* Item's value list initialized prior to calling expand(). */
- LIST * values;
-
- /* Internal data initialized and used inside expand(). */
- LISTITER current; /* Currently used value. */
- int size; /* Concatenated string length prior to concatenating the
- * item's current value.
- */
-} expansion_item;
-
-static LIST * expand( expansion_item * items, int const length )
-{
- LIST * result = L0;
- string buf[ 1 ];
- int size = 0;
- int i;
-
- assert( length > 0 );
- for ( i = 0; i < length; ++i )
- {
- LISTITER iter = list_begin( items[ i ].values );
- LISTITER const end = list_end( items[ i ].values );
-
- /* If any of the items has no values - the result is an empty list. */
- if ( iter == end ) return L0;
-
- /* Set each item's 'current' to its first listed value. This indicates
- * each item's next value to be used when constructing the list of all
- * possible concatenated values.
- */
- items[ i ].current = iter;
-
- /* Calculate the longest concatenated string length - to know how much
- * memory we need to allocate as a buffer for holding the concatenated
- * strings.
- */
- {
- int max = 0;
- for ( ; iter != end; iter = list_next( iter ) )
- {
- int const len = strlen( object_str( list_item( iter ) ) );
- if ( len > max ) max = len;
- }
- size += max;
- }
- }
-
- string_new( buf );
- string_reserve( buf, size );
-
- i = 0;
- while ( i >= 0 )
- {
- for ( ; i < length; ++i )
- {
- items[ i ].size = buf->size;
- string_append( buf, object_str( list_item( items[ i ].current ) ) );
- }
- result = list_push_back( result, object_new( buf->value ) );
- while ( --i >= 0 )
- {
- if ( list_next( items[ i ].current ) != list_end( items[ i ].values
- ) )
- {
- items[ i ].current = list_next( items[ i ].current );
- string_truncate( buf, items[ i ].size );
- break;
- }
- else
- items[ i ].current = list_begin( items[ i ].values );
- }
- }
-
- string_free( buf );
- return result;
-}
-
-static void combine_strings( STACK * s, int n, string * out )
-{
- int i;
- for ( i = 0; i < n; ++i )
- {
- LIST * const values = stack_pop( s );
- LISTITER iter = list_begin( values );
- LISTITER const end = list_end( values );
- if ( iter != end )
- {
- string_append( out, object_str( list_item( iter ) ) );
- for ( iter = list_next( iter ); iter != end; iter = list_next( iter
- ) )
- {
- string_push_back( out, ' ' );
- string_append( out, object_str( list_item( iter ) ) );
- }
- list_free( values );
- }
- }
-}
-
-struct dynamic_array
-{
- int size;
- int capacity;
- void * data;
-};
-
-static void dynamic_array_init( struct dynamic_array * array )
-{
- array->size = 0;
- array->capacity = 0;
- array->data = 0;
-}
-
-static void dynamic_array_free( struct dynamic_array * array )
-{
- BJAM_FREE( array->data );
-}
-
-static void dynamic_array_push_impl( struct dynamic_array * const array,
- void const * const value, int const unit_size )
-{
- if ( array->capacity == 0 )
- {
- array->capacity = 2;
- array->data = BJAM_MALLOC( array->capacity * unit_size );
- }
- else if ( array->capacity == array->size )
- {
- void * new_data;
- array->capacity *= 2;
- new_data = BJAM_MALLOC( array->capacity * unit_size );
- memcpy( new_data, array->data, array->size * unit_size );
- BJAM_FREE( array->data );
- array->data = new_data;
- }
- memcpy( (char *)array->data + array->size * unit_size, value, unit_size );
- ++array->size;
-}
-
-#define dynamic_array_push( array, value ) (dynamic_array_push_impl(array, &value, sizeof(value)))
-#define dynamic_array_at( type, array, idx ) (((type *)(array)->data)[idx])
-
-
-/*
- * struct compiler
- */
-
-struct label_info
-{
- int absolute_position;
- struct dynamic_array uses[ 1 ];
-};
-
-struct stored_rule
-{
- OBJECT * name;
- PARSE * parse;
- int num_arguments;
- struct arg_list * arguments;
- int local;
-};
-
-typedef struct compiler
-{
- struct dynamic_array code[ 1 ];
- struct dynamic_array constants[ 1 ];
- struct dynamic_array labels[ 1 ];
- struct dynamic_array rules[ 1 ];
- struct dynamic_array actions[ 1 ];
-} compiler;
-
-static void compiler_init( compiler * c )
-{
- dynamic_array_init( c->code );
- dynamic_array_init( c->constants );
- dynamic_array_init( c->labels );
- dynamic_array_init( c->rules );
- dynamic_array_init( c->actions );
-}
-
-static void compiler_free( compiler * c )
-{
- int i;
- dynamic_array_free( c->actions );
- dynamic_array_free( c->rules );
- for ( i = 0; i < c->labels->size; ++i )
- dynamic_array_free( dynamic_array_at( struct label_info, c->labels, i
- ).uses );
- dynamic_array_free( c->labels );
- dynamic_array_free( c->constants );
- dynamic_array_free( c->code );
-}
-
-static void compile_emit_instruction( compiler * c, instruction instr )
-{
- dynamic_array_push( c->code, instr );
-}
-
-static int compile_new_label( compiler * c )
-{
- int result = c->labels->size;
- struct label_info info;
- info.absolute_position = -1;
- dynamic_array_init( info.uses );
- dynamic_array_push( c->labels, info );
- return result;
-}
-
-static void compile_set_label( compiler * c, int label )
-{
- struct label_info * const l = &dynamic_array_at( struct label_info,
- c->labels, label );
- int const pos = c->code->size;
- int i;
- assert( l->absolute_position == -1 );
- l->absolute_position = pos;
- for ( i = 0; i < l->uses->size; ++i )
- {
- int id = dynamic_array_at( int, l->uses, i );
- int offset = (int)( pos - id - 1 );
- dynamic_array_at( instruction, c->code, id ).arg = offset;
- }
-}
-
-static void compile_emit( compiler * c, unsigned int op_code, int arg )
-{
- instruction instr;
- instr.op_code = op_code;
- instr.arg = arg;
- compile_emit_instruction( c, instr );
-}
-
-static void compile_emit_branch( compiler * c, unsigned int op_code, int label )
-{
- struct label_info * const l = &dynamic_array_at( struct label_info,
- c->labels, label );
- int const pos = c->code->size;
- instruction instr;
- instr.op_code = op_code;
- if ( l->absolute_position == -1 )
- {
- instr.arg = 0;
- dynamic_array_push( l->uses, pos );
- }
- else
- instr.arg = (int)( l->absolute_position - pos - 1 );
- compile_emit_instruction( c, instr );
-}
-
-static int compile_emit_constant( compiler * c, OBJECT * value )
-{
- OBJECT * copy = object_copy( value );
- dynamic_array_push( c->constants, copy );
- return c->constants->size - 1;
-}
-
-static int compile_emit_rule( compiler * c, OBJECT * name, PARSE * parse,
- int num_arguments, struct arg_list * arguments, int local )
-{
- struct stored_rule rule;
- rule.name = object_copy( name );
- rule.parse = parse;
- rule.num_arguments = num_arguments;
- rule.arguments = arguments;
- rule.local = local;
- dynamic_array_push( c->rules, rule );
- return (int)( c->rules->size - 1 );
-}
-
-static int compile_emit_actions( compiler * c, PARSE * parse )
-{
- SUBACTION a;
- a.name = object_copy( parse->string );
- a.command = function_compile_actions( object_str( parse->string1 ),
- parse->file, parse->line );
- a.flags = parse->num;
- dynamic_array_push( c->actions, a );
- return (int)( c->actions->size - 1 );
-}
-
-static JAM_FUNCTION * compile_to_function( compiler * c )
-{
- JAM_FUNCTION * const result = BJAM_MALLOC( sizeof( JAM_FUNCTION ) );
- int i;
- result->base.type = FUNCTION_JAM;
- result->base.reference_count = 1;
- result->base.formal_arguments = 0;
- result->base.num_formal_arguments = 0;
-
- result->base.rulename = 0;
-
- result->code_size = c->code->size;
- result->code = BJAM_MALLOC( c->code->size * sizeof( instruction ) );
- memcpy( result->code, c->code->data, c->code->size * sizeof( instruction ) );
-
- result->constants = BJAM_MALLOC( c->constants->size * sizeof( OBJECT * ) );
- memcpy( result->constants, c->constants->data, c->constants->size * sizeof(
- OBJECT * ) );
- result->num_constants = c->constants->size;
-
- result->num_subfunctions = c->rules->size;
- result->functions = BJAM_MALLOC( c->rules->size * sizeof( SUBFUNCTION ) );
- for ( i = 0; i < c->rules->size; ++i )
- {
- struct stored_rule * const rule = &dynamic_array_at( struct stored_rule,
- c->rules, i );
- result->functions[ i ].name = rule->name;
- result->functions[ i ].code = function_compile( rule->parse );
- result->functions[ i ].code->num_formal_arguments = rule->num_arguments;
- result->functions[ i ].code->formal_arguments = rule->arguments;
- result->functions[ i ].local = rule->local;
- }
-
- result->actions = BJAM_MALLOC( c->actions->size * sizeof( SUBACTION ) );
- memcpy( result->actions, c->actions->data, c->actions->size * sizeof(
- SUBACTION ) );
- result->num_subactions = c->actions->size;
-
- result->generic = 0;
-
- result->file = 0;
- result->line = -1;
-
- return result;
-}
-
-
-/*
- * Parsing of variable expansions
- */
-
-typedef struct VAR_PARSE_GROUP
-{
- struct dynamic_array elems[ 1 ];
-} VAR_PARSE_GROUP;
-
-typedef struct VAR_PARSE_ACTIONS
-{
- struct dynamic_array elems[ 1 ];
-} VAR_PARSE_ACTIONS;
-
-#define VAR_PARSE_TYPE_VAR 0
-#define VAR_PARSE_TYPE_STRING 1
-#define VAR_PARSE_TYPE_FILE 2
-
-typedef struct _var_parse
-{
- int type; /* string, variable or file */
-} VAR_PARSE;
-
-typedef struct
-{
- VAR_PARSE base;
- VAR_PARSE_GROUP * name;
- VAR_PARSE_GROUP * subscript;
- struct dynamic_array modifiers[ 1 ];
-} VAR_PARSE_VAR;
-
-typedef struct
-{
- VAR_PARSE base;
- OBJECT * s;
-} VAR_PARSE_STRING;
-
-typedef struct
-{
- VAR_PARSE base;
- struct dynamic_array filename[ 1 ];
- struct dynamic_array contents[ 1 ];
-} VAR_PARSE_FILE;
-
-static void var_parse_free( VAR_PARSE * );
-
-
-/*
- * VAR_PARSE_GROUP
- */
-
-static VAR_PARSE_GROUP * var_parse_group_new()
-{
- VAR_PARSE_GROUP * const result = BJAM_MALLOC( sizeof( VAR_PARSE_GROUP ) );
- dynamic_array_init( result->elems );
- return result;
-}
-
-static void var_parse_group_free( VAR_PARSE_GROUP * group )
-{
- int i;
- for ( i = 0; i < group->elems->size; ++i )
- var_parse_free( dynamic_array_at( VAR_PARSE *, group->elems, i ) );
- dynamic_array_free( group->elems );
- BJAM_FREE( group );
-}
-
-static void var_parse_group_add( VAR_PARSE_GROUP * group, VAR_PARSE * elem )
-{
- dynamic_array_push( group->elems, elem );
-}
-
-static void var_parse_group_maybe_add_constant( VAR_PARSE_GROUP * group,
- char const * start, char const * end )
-{
- if ( start != end )
- {
- string buf[ 1 ];
- VAR_PARSE_STRING * const value = (VAR_PARSE_STRING *)BJAM_MALLOC(
- sizeof(VAR_PARSE_STRING) );
- value->base.type = VAR_PARSE_TYPE_STRING;
- string_new( buf );
- string_append_range( buf, start, end );
- value->s = object_new( buf->value );
- string_free( buf );
- var_parse_group_add( group, (VAR_PARSE *)value );
- }
-}
-
-VAR_PARSE_STRING * var_parse_group_as_literal( VAR_PARSE_GROUP * group )
-{
- if ( group->elems->size == 1 )
- {
- VAR_PARSE * result = dynamic_array_at( VAR_PARSE *, group->elems, 0 );
- if ( result->type == VAR_PARSE_TYPE_STRING )
- return (VAR_PARSE_STRING *)result;
- }
- return 0;
-}
-
-
-/*
- * VAR_PARSE_ACTIONS
- */
-
-static VAR_PARSE_ACTIONS * var_parse_actions_new()
-{
- VAR_PARSE_ACTIONS * const result = (VAR_PARSE_ACTIONS *)BJAM_MALLOC(
- sizeof(VAR_PARSE_ACTIONS) );
- dynamic_array_init( result->elems );
- return result;
-}
-
-static void var_parse_actions_free( VAR_PARSE_ACTIONS * actions )
-{
- int i;
- for ( i = 0; i < actions->elems->size; ++i )
- var_parse_group_free( dynamic_array_at( VAR_PARSE_GROUP *,
- actions->elems, i ) );
- dynamic_array_free( actions->elems );
- BJAM_FREE( actions );
-}
-
-
-/*
- * VAR_PARSE_VAR
- */
-
-static VAR_PARSE_VAR * var_parse_var_new()
-{
- VAR_PARSE_VAR * result = BJAM_MALLOC( sizeof( VAR_PARSE_VAR ) );
- result->base.type = VAR_PARSE_TYPE_VAR;
- result->name = var_parse_group_new();
- result->subscript = 0;
- dynamic_array_init( result->modifiers );
- return result;
-}
-
-static void var_parse_var_free( VAR_PARSE_VAR * var )
-{
- int i;
- var_parse_group_free( var->name );
- if ( var->subscript )
- var_parse_group_free( var->subscript );
- for ( i = 0; i < var->modifiers->size; ++i )
- var_parse_group_free( dynamic_array_at( VAR_PARSE_GROUP *,
- var->modifiers, i ) );
- dynamic_array_free( var->modifiers );
- BJAM_FREE( var );
-}
-
-static VAR_PARSE_GROUP * var_parse_var_new_modifier( VAR_PARSE_VAR * var )
-{
- VAR_PARSE_GROUP * result = var_parse_group_new();
- dynamic_array_push( var->modifiers, result );
- return result;
-}
-
-
-/*
- * VAR_PARSE_STRING
- */
-
-static void var_parse_string_free( VAR_PARSE_STRING * string )
-{
- object_free( string->s );
- BJAM_FREE( string );
-}
-
-
-/*
- * VAR_PARSE_FILE
- */
-
-static VAR_PARSE_FILE * var_parse_file_new( void )
-{
- VAR_PARSE_FILE * const result = (VAR_PARSE_FILE *)BJAM_MALLOC( sizeof(
- VAR_PARSE_FILE ) );
- result->base.type = VAR_PARSE_TYPE_FILE;
- dynamic_array_init( result->filename );
- dynamic_array_init( result->contents );
- return result;
-}
-
-static void var_parse_file_free( VAR_PARSE_FILE * file )
-{
- int i;
- for ( i = 0; i < file->filename->size; ++i )
- var_parse_group_free( dynamic_array_at( VAR_PARSE_GROUP *,
- file->filename, i ) );
- dynamic_array_free( file->filename );
- for ( i = 0; i < file->contents->size; ++i )
- var_parse_group_free( dynamic_array_at( VAR_PARSE_GROUP *,
- file->contents, i ) );
- dynamic_array_free( file->contents );
- BJAM_FREE( file );
-}
-
-
-/*
- * VAR_PARSE
- */
-
-static void var_parse_free( VAR_PARSE * parse )
-{
- switch ( parse->type )
- {
- case VAR_PARSE_TYPE_VAR:
- var_parse_var_free( (VAR_PARSE_VAR *)parse );
- break;
-
- case VAR_PARSE_TYPE_STRING:
- var_parse_string_free( (VAR_PARSE_STRING *)parse );
- break;
-
- case VAR_PARSE_TYPE_FILE:
- var_parse_file_free( (VAR_PARSE_FILE *)parse );
- break;
-
- default:
- assert( !"Invalid type" );
- }
-}
-
-
-/*
- * Compile VAR_PARSE
- */
-
-static void var_parse_group_compile( VAR_PARSE_GROUP const * parse,
- compiler * c );
-
-static void var_parse_var_compile( VAR_PARSE_VAR const * parse, compiler * c )
-{
- int expand_name = 0;
- int is_get_grist = 0;
- int has_modifiers = 0;
- /* Special case common modifiers */
- if ( parse->modifiers->size == 1 )
- {
- VAR_PARSE_GROUP * mod = dynamic_array_at( VAR_PARSE_GROUP *, parse->modifiers, 0 );
- if ( mod->elems->size == 1 )
- {
- VAR_PARSE * mod1 = dynamic_array_at( VAR_PARSE *, mod->elems, 0 );
- if ( mod1->type == VAR_PARSE_TYPE_STRING )
- {
- OBJECT * s = ( (VAR_PARSE_STRING *)mod1 )->s;
- if ( ! strcmp ( object_str( s ), "G" ) )
- {
- is_get_grist = 1;
- }
- }
- }
- }
- /* If there are modifiers, emit them in reverse order. */
- if ( parse->modifiers->size > 0 && !is_get_grist )
- {
- int i;
- has_modifiers = 1;
- for ( i = 0; i < parse->modifiers->size; ++i )
- var_parse_group_compile( dynamic_array_at( VAR_PARSE_GROUP *,
- parse->modifiers, parse->modifiers->size - i - 1 ), c );
- }
-
- /* If there is a subscript, emit it. */
- if ( parse->subscript )
- var_parse_group_compile( parse->subscript, c );
-
- /* If the variable name is empty, look it up. */
- if ( parse->name->elems->size == 0 )
- compile_emit( c, INSTR_PUSH_VAR, compile_emit_constant( c,
- constant_empty ) );
- /* If the variable name does not need to be expanded, look it up. */
- else if ( parse->name->elems->size == 1 && dynamic_array_at( VAR_PARSE *,
- parse->name->elems, 0 )->type == VAR_PARSE_TYPE_STRING )
- {
- OBJECT * const name = ( (VAR_PARSE_STRING *)dynamic_array_at(
- VAR_PARSE *, parse->name->elems, 0 ) )->s;
- int const idx = get_argument_index( object_str( name ) );
- if ( idx != -1 )
- compile_emit( c, INSTR_PUSH_ARG, idx );
- else
- compile_emit( c, INSTR_PUSH_VAR, compile_emit_constant( c, name ) );
- }
- /* Otherwise, push the var names and use the group instruction. */
- else
- {
- var_parse_group_compile( parse->name, c );
- expand_name = 1;
- }
-
- /** Select the instruction for expanding the variable. */
- if ( !has_modifiers && !parse->subscript && !expand_name )
- ;
- else if ( !has_modifiers && !parse->subscript && expand_name )
- compile_emit( c, INSTR_PUSH_GROUP, 0 );
- else if ( !has_modifiers && parse->subscript && !expand_name )
- compile_emit( c, INSTR_APPLY_INDEX, 0 );
- else if ( !has_modifiers && parse->subscript && expand_name )
- compile_emit( c, INSTR_APPLY_INDEX_GROUP, 0 );
- else if ( has_modifiers && !parse->subscript && !expand_name )
- compile_emit( c, INSTR_APPLY_MODIFIERS, parse->modifiers->size );
- else if ( has_modifiers && !parse->subscript && expand_name )
- compile_emit( c, INSTR_APPLY_MODIFIERS_GROUP, parse->modifiers->size );
- else if ( has_modifiers && parse->subscript && !expand_name )
- compile_emit( c, INSTR_APPLY_INDEX_MODIFIERS, parse->modifiers->size );
- else if ( has_modifiers && parse->subscript && expand_name )
- compile_emit( c, INSTR_APPLY_INDEX_MODIFIERS_GROUP,
- parse->modifiers->size );
-
- /* Now apply any special modifiers */
- if ( is_get_grist )
- {
- compile_emit( c, INSTR_GET_GRIST, 0 );
- }
-}
-
-static void var_parse_string_compile( VAR_PARSE_STRING const * parse,
- compiler * c )
-{
- compile_emit( c, INSTR_PUSH_CONSTANT, compile_emit_constant( c, parse->s )
- );
-}
-
-static void var_parse_file_compile( VAR_PARSE_FILE const * parse, compiler * c )
-{
- int i;
- for ( i = 0; i < parse->filename->size; ++i )
- var_parse_group_compile( dynamic_array_at( VAR_PARSE_GROUP *,
- parse->filename, parse->filename->size - i - 1 ), c );
- compile_emit( c, INSTR_APPEND_STRINGS, parse->filename->size );
- for ( i = 0; i < parse->contents->size; ++i )
- var_parse_group_compile( dynamic_array_at( VAR_PARSE_GROUP *,
- parse->contents, parse->contents->size - i - 1 ), c );
- compile_emit( c, INSTR_WRITE_FILE, parse->contents->size );
-}
-
-static void var_parse_compile( VAR_PARSE const * parse, compiler * c )
-{
- switch ( parse->type )
- {
- case VAR_PARSE_TYPE_VAR:
- var_parse_var_compile( (VAR_PARSE_VAR const *)parse, c );
- break;
-
- case VAR_PARSE_TYPE_STRING:
- var_parse_string_compile( (VAR_PARSE_STRING const *)parse, c );
- break;
-
- case VAR_PARSE_TYPE_FILE:
- var_parse_file_compile( (VAR_PARSE_FILE const *)parse, c );
- break;
-
- default:
- assert( !"Unknown var parse type." );
- }
-}
-
-static void var_parse_group_compile( VAR_PARSE_GROUP const * parse, compiler * c
- )
-{
- /* Emit the elements in reverse order. */
- int i;
- for ( i = 0; i < parse->elems->size; ++i )
- var_parse_compile( dynamic_array_at( VAR_PARSE *, parse->elems,
- parse->elems->size - i - 1 ), c );
- /* If there are no elements, emit an empty string. */
- if ( parse->elems->size == 0 )
- compile_emit( c, INSTR_PUSH_CONSTANT, compile_emit_constant( c,
- constant_empty ) );
- /* If there is more than one element, combine them. */
- if ( parse->elems->size > 1 )
- compile_emit( c, INSTR_COMBINE_STRINGS, parse->elems->size );
-}
-
-static void var_parse_actions_compile( VAR_PARSE_ACTIONS const * actions,
- compiler * c )
-{
- int i;
- for ( i = 0; i < actions->elems->size; ++i )
- var_parse_group_compile( dynamic_array_at( VAR_PARSE_GROUP *,
- actions->elems, actions->elems->size - i - 1 ), c );
- compile_emit( c, INSTR_OUTPUT_STRINGS, actions->elems->size );
-}
-
-
-/*
- * Parse VAR_PARSE_VAR
- */
-
-static VAR_PARSE * parse_at_file( char const * start, char const * mid,
- char const * end );
-static VAR_PARSE * parse_variable( char const * * string );
-static int try_parse_variable( char const * * s_, char const * * string,
- VAR_PARSE_GROUP * out );
-static void balance_parentheses( char const * * s_, char const * * string,
- VAR_PARSE_GROUP * out );
-static void parse_var_string( char const * first, char const * last,
- struct dynamic_array * out );
-
-
-/*
- * Parses a string that can contain variables to expand.
- */
-
-static VAR_PARSE_GROUP * parse_expansion( char const * * string )
-{
- VAR_PARSE_GROUP * result = var_parse_group_new();
- char const * s = *string;
- for ( ; ; )
- {
- if ( try_parse_variable( &s, string, result ) ) {}
- else if ( s[ 0 ] == '\0' )
- {
- var_parse_group_maybe_add_constant( result, *string, s );
- return result;
- }
- else
- ++s;
- }
-}
-
-static VAR_PARSE_ACTIONS * parse_actions( char const * string )
-{
- VAR_PARSE_ACTIONS * const result = var_parse_actions_new();
- parse_var_string( string, string + strlen( string ), result->elems );
- return result;
-}
-
-/*
- * Checks whether the string a *s_ starts with a variable expansion "$(".
- * *string should point to the first unemitted character before *s. If *s_
- * starts with variable expansion, appends elements to out up to the closing
- * ")", and adjusts *s_ and *string to point to next character. Returns 1 if s_
- * starts with a variable, 0 otherwise.
- */
-
-static int try_parse_variable( char const * * s_, char const * * string,
- VAR_PARSE_GROUP * out )
-{
- char const * s = *s_;
- if ( s[ 0 ] == '$' && s[ 1 ] == '(' )
- {
- var_parse_group_maybe_add_constant( out, *string, s );
- s += 2;
- var_parse_group_add( out, parse_variable( &s ) );
- *string = s;
- *s_ = s;
- return 1;
- }
- if ( s[ 0 ] == '@' && s[ 1 ] == '(' )
- {
- int depth = 1;
- char const * ine;
- char const * split = 0;
- var_parse_group_maybe_add_constant( out, *string, s );
- s += 2;
- ine = s;
-
- /* Scan the content of the response file @() section. */
- while ( *ine && ( depth > 0 ) )
- {
- switch ( *ine )
- {
- case '(': ++depth; break;
- case ')': --depth; break;
- case ':':
- if ( ( depth == 1 ) && ( ine[ 1 ] == 'E' ) && ( ine[ 2 ] == '='
- ) )
- split = ine;
- break;
- }
- ++ine;
- }
-
- if ( !split || depth )
- return 0;
-
- var_parse_group_add( out, parse_at_file( s, split, ine - 1 ) );
- *string = ine;
- *s_ = ine;
- return 1;
- }
- return 0;
-}
-
-
-static char const * current_file = "";
-static int current_line;
-
-static void parse_error( char const * message )
-{
- printf( "%s:%d: %s\n", current_file, current_line, message );
-}
-
-
-/*
- * Parses a single variable up to the closing ")" and adjusts *string to point
- * to the next character. *string should point to the character immediately
- * after the initial "$(".
- */
-
-static VAR_PARSE * parse_variable( char const * * string )
-{
- VAR_PARSE_VAR * const result = var_parse_var_new();
- VAR_PARSE_GROUP * const name = result->name;
- char const * s = *string;
- for ( ; ; )
- {
- if ( try_parse_variable( &s, string, name ) ) {}
- else if ( s[ 0 ] == ':' )
- {
- VAR_PARSE_GROUP * mod;
- var_parse_group_maybe_add_constant( name, *string, s );
- ++s;
- *string = s;
- mod = var_parse_var_new_modifier( result );
- for ( ; ; )
- {
- if ( try_parse_variable( &s, string, mod ) ) {}
- else if ( s[ 0 ] == ')' )
- {
- var_parse_group_maybe_add_constant( mod, *string, s );
- *string = ++s;
- return (VAR_PARSE *)result;
- }
- else if ( s[ 0 ] == '(' )
- {
- ++s;
- balance_parentheses( &s, string, mod );
- }
- else if ( s[ 0 ] == ':' )
- {
- var_parse_group_maybe_add_constant( mod, *string, s );
- *string = ++s;
- mod = var_parse_var_new_modifier( result );
- }
- else if ( s[ 0 ] == '[' )
- {
- parse_error("unexpected subscript");
- ++s;
- }
- else if ( s[ 0 ] == '\0' )
- {
- parse_error( "unbalanced parentheses" );
- var_parse_group_maybe_add_constant( mod, *string, s );
- *string = s;
- return (VAR_PARSE *)result;
- }
- else
- ++s;
- }
- }
- else if ( s[ 0 ] == '[' )
- {
- VAR_PARSE_GROUP * subscript = var_parse_group_new();
- result->subscript = subscript;
- var_parse_group_maybe_add_constant( name, *string, s );
- *string = ++s;
- for ( ; ; )
- {
- if ( try_parse_variable( &s, string, subscript ) ) {}
- else if ( s[ 0 ] == ']' )
- {
- var_parse_group_maybe_add_constant( subscript, *string, s );
- *string = ++s;
- if ( s[ 0 ] != ')' && s[ 0 ] != ':' && s[ 0 ] != '\0' )
- parse_error( "unexpected text following []" );
- break;
- }
- else if ( isdigit( s[ 0 ] ) || s[ 0 ] == '-' )
- {
- ++s;
- }
- else if ( s[ 0 ] == '\0' )
- {
- parse_error( "malformed subscript" );
- break;
- }
- else
- {
- parse_error( "malformed subscript" );
- ++s;
- }
- }
- }
- else if ( s[ 0 ] == ')' )
- {
- var_parse_group_maybe_add_constant( name, *string, s );
- *string = ++s;
- return (VAR_PARSE *)result;
- }
- else if ( s[ 0 ] == '(' )
- {
- ++s;
- balance_parentheses( &s, string, name );
- }
- else if ( s[ 0 ] == '\0' )
- {
- parse_error( "unbalanced parentheses" );
- var_parse_group_maybe_add_constant( name, *string, s );
- *string = s;
- return (VAR_PARSE *)result;
- }
- else
- ++s;
- }
-}
-
-static void parse_var_string( char const * first, char const * last,
- struct dynamic_array * out )
-{
- char const * saved = first;
- while ( first != last )
- {
- /* Handle whitespace. */
- while ( first != last && isspace( *first ) ) ++first;
- if ( saved != first )
- {
- VAR_PARSE_GROUP * const group = var_parse_group_new();
- var_parse_group_maybe_add_constant( group, saved, first );
- saved = first;
- dynamic_array_push( out, group );
- }
- if ( first == last ) break;
-
- /* Handle non-whitespace */
- {
- VAR_PARSE_GROUP * group = var_parse_group_new();
- for ( ; ; )
- {
- if ( first == last || isspace( *first ) )
- {
- var_parse_group_maybe_add_constant( group, saved, first );
- saved = first;
- break;
- }
- if ( try_parse_variable( &first, &saved, group ) )
- assert( first <= last );
- else
- ++first;
- }
- dynamic_array_push( out, group );
- }
- }
-}
-
-/*
- * start should point to the character immediately following the opening "@(",
- * mid should point to the ":E=", and end should point to the closing ")".
- */
-
-static VAR_PARSE * parse_at_file( char const * start, char const * mid,
- char const * end )
-{
- VAR_PARSE_FILE * result = var_parse_file_new();
- parse_var_string( start, mid, result->filename );
- parse_var_string( mid + 3, end, result->contents );
- return (VAR_PARSE *)result;
-}
-
-/*
- * Given that *s_ points to the character after a "(", parses up to the matching
- * ")". *string should point to the first unemitted character before *s_.
- *
- * When the function returns, *s_ will point to the character after the ")", and
- * *string will point to the first unemitted character before *s_. The range
- * from *string to *s_ does not contain any variables that need to be expanded.
- */
-
-void balance_parentheses( char const * * s_, char const * * string,
- VAR_PARSE_GROUP * out)
-{
- int depth = 1;
- char const * s = *s_;
- for ( ; ; )
- {
- if ( try_parse_variable( &s, string, out ) ) { }
- else if ( s[ 0 ] == ':' || s[ 0 ] == '[' )
- {
- parse_error( "unbalanced parentheses" );
- ++s;
- }
- else if ( s[ 0 ] == '\0' )
- {
- parse_error( "unbalanced parentheses" );
- break;
- }
- else if ( s[ 0 ] == ')' )
- {
- ++s;
- if ( --depth == 0 ) break;
- }
- else if ( s[ 0 ] == '(' )
- {
- ++depth;
- ++s;
- }
- else
- ++s;
- }
- *s_ = s;
-}
-
-
-/*
- * Main compile.
- */
-
-#define RESULT_STACK 0
-#define RESULT_RETURN 1
-#define RESULT_NONE 2
-
-static void compile_parse( PARSE * parse, compiler * c, int result_location );
-static struct arg_list * arg_list_compile( PARSE * parse, int * num_arguments );
-
-static void compile_condition( PARSE * parse, compiler * c, int branch_true, int label )
-{
- assert( parse->type == PARSE_EVAL );
- switch ( parse->num )
- {
- case EXPR_EXISTS:
- compile_parse( parse->left, c, RESULT_STACK );
- if ( branch_true )
- compile_emit_branch( c, INSTR_JUMP_NOT_EMPTY, label );
- else
- compile_emit_branch( c, INSTR_JUMP_EMPTY, label );
- break;
-
- case EXPR_EQUALS:
- compile_parse( parse->left, c, RESULT_STACK );
- compile_parse( parse->right, c, RESULT_STACK );
- if ( branch_true )
- compile_emit_branch( c, INSTR_JUMP_EQ, label );
- else
- compile_emit_branch( c, INSTR_JUMP_NE, label );
- break;
-
- case EXPR_NOTEQ:
- compile_parse( parse->left, c, RESULT_STACK );
- compile_parse( parse->right, c, RESULT_STACK );
- if ( branch_true )
- compile_emit_branch( c, INSTR_JUMP_NE, label );
- else
- compile_emit_branch( c, INSTR_JUMP_EQ, label );
- break;
-
- case EXPR_LESS:
- compile_parse( parse->left, c, RESULT_STACK );
- compile_parse( parse->right, c, RESULT_STACK );
- if ( branch_true )
- compile_emit_branch( c, INSTR_JUMP_LT, label );
- else
- compile_emit_branch( c, INSTR_JUMP_GE, label );
- break;
-
- case EXPR_LESSEQ:
- compile_parse( parse->left, c, RESULT_STACK );
- compile_parse( parse->right, c, RESULT_STACK );
- if ( branch_true )
- compile_emit_branch( c, INSTR_JUMP_LE, label );
- else
- compile_emit_branch( c, INSTR_JUMP_GT, label );
- break;
-
- case EXPR_MORE:
- compile_parse( parse->left, c, RESULT_STACK );
- compile_parse( parse->right, c, RESULT_STACK );
- if ( branch_true )
- compile_emit_branch( c, INSTR_JUMP_GT, label );
- else
- compile_emit_branch( c, INSTR_JUMP_LE, label );
- break;
-
- case EXPR_MOREEQ:
- compile_parse( parse->left, c, RESULT_STACK );
- compile_parse( parse->right, c, RESULT_STACK );
- if ( branch_true )
- compile_emit_branch( c, INSTR_JUMP_GE, label );
- else
- compile_emit_branch( c, INSTR_JUMP_LT, label );
- break;
-
- case EXPR_IN:
- compile_parse( parse->left, c, RESULT_STACK );
- compile_parse( parse->right, c, RESULT_STACK );
- if ( branch_true )
- compile_emit_branch( c, INSTR_JUMP_IN, label );
- else
- compile_emit_branch( c, INSTR_JUMP_NOT_IN, label );
- break;
-
- case EXPR_AND:
- if ( branch_true )
- {
- int f = compile_new_label( c );
- compile_condition( parse->left, c, 0, f );
- compile_condition( parse->right, c, 1, label );
- compile_set_label( c, f );
- }
- else
- {
- compile_condition( parse->left, c, 0, label );
- compile_condition( parse->right, c, 0, label );
- }
- break;
-
- case EXPR_OR:
- if ( branch_true )
- {
- compile_condition( parse->left, c, 1, label );
- compile_condition( parse->right, c, 1, label );
- }
- else
- {
- int t = compile_new_label( c );
- compile_condition( parse->left, c, 1, t );
- compile_condition( parse->right, c, 0, label );
- compile_set_label( c, t );
- }
- break;
-
- case EXPR_NOT:
- compile_condition( parse->left, c, !branch_true, label );
- break;
- }
-}
-
-static void adjust_result( compiler * c, int actual_location,
- int desired_location )
-{
- if ( actual_location == desired_location )
- ;
- else if ( actual_location == RESULT_STACK && desired_location == RESULT_RETURN )
- compile_emit( c, INSTR_SET_RESULT, 0 );
- else if ( actual_location == RESULT_STACK && desired_location == RESULT_NONE )
- compile_emit( c, INSTR_POP, 0 );
- else if ( actual_location == RESULT_RETURN && desired_location == RESULT_STACK )
- compile_emit( c, INSTR_PUSH_RESULT, 0 );
- else if ( actual_location == RESULT_RETURN && desired_location == RESULT_NONE )
- ;
- else if ( actual_location == RESULT_NONE && desired_location == RESULT_STACK )
- compile_emit( c, INSTR_PUSH_EMPTY, 0 );
- else if ( actual_location == RESULT_NONE && desired_location == RESULT_RETURN )
- {
- compile_emit( c, INSTR_PUSH_EMPTY, 0 );
- compile_emit( c, INSTR_SET_RESULT, 0 );
- }
- else
- assert( !"invalid result location" );
-}
-
-static char const * parse_type( PARSE * parse )
-{
- switch ( parse->type )
- {
- case PARSE_APPEND: return "append";
- case PARSE_EVAL: return "eval";
- case PARSE_RULES: return "rules";
- default: return "unknown";
- }
-}
-
-static void compile_append_chain( PARSE * parse, compiler * c )
-{
- assert( parse->type == PARSE_APPEND );
- if ( parse->left->type == PARSE_NULL )
- compile_parse( parse->right, c, RESULT_STACK );
- else
- {
- if ( parse->left->type == PARSE_APPEND )
- compile_append_chain( parse->left, c );
- else
- compile_parse( parse->left, c, RESULT_STACK );
- compile_parse( parse->right, c, RESULT_STACK );
- compile_emit( c, INSTR_PUSH_APPEND, 0 );
- }
-}
-
-static void compile_parse( PARSE * parse, compiler * c, int result_location )
-{
- if ( parse->type == PARSE_APPEND )
- {
- compile_append_chain( parse, c );
- adjust_result( c, RESULT_STACK, result_location );
- }
- else if ( parse->type == PARSE_EVAL )
- {
- /* FIXME: This is only needed because of the bizarre parsing of
- * conditions.
- */
- if ( parse->num == EXPR_EXISTS )
- compile_parse( parse->left, c, result_location );
- else
- {
- int f = compile_new_label( c );
- int end = compile_new_label( c );
-
- printf( "%s:%d: Conditional used as list (check operator "
- "precedence).\n", object_str( parse->file ), parse->line );
-
- /* Emit the condition */
- compile_condition( parse, c, 0, f );
- compile_emit( c, INSTR_PUSH_CONSTANT, compile_emit_constant( c,
- constant_true ) );
- compile_emit_branch( c, INSTR_JUMP, end );
- compile_set_label( c, f );
- compile_emit( c, INSTR_PUSH_EMPTY, 0 );
- compile_set_label( c, end );
- adjust_result( c, RESULT_STACK, result_location );
- }
- }
- else if ( parse->type == PARSE_FOREACH )
- {
- int var = compile_emit_constant( c, parse->string );
- int top = compile_new_label( c );
- int end = compile_new_label( c );
-
- /*
- * Evaluate the list.
- */
- compile_parse( parse->left, c, RESULT_STACK );
-
- /* Localize the loop variable */
- if ( parse->num )
- {
- compile_emit( c, INSTR_PUSH_EMPTY, 0 );
- compile_emit( c, INSTR_PUSH_LOCAL, var );
- compile_emit( c, INSTR_SWAP, 1 );
- }
-
- compile_emit( c, INSTR_FOR_INIT, 0 );
- compile_set_label( c, top );
- compile_emit_branch( c, INSTR_FOR_LOOP, end );
- compile_emit( c, INSTR_SET, var );
-
- /* Run the loop body */
- compile_parse( parse->right, c, RESULT_NONE );
-
- compile_emit_branch( c, INSTR_JUMP, top );
- compile_set_label( c, end );
-
- if ( parse->num )
- compile_emit( c, INSTR_POP_LOCAL, var );
-
- adjust_result( c, RESULT_NONE, result_location);
- }
- else if ( parse->type == PARSE_IF )
- {
- int f = compile_new_label( c );
- /* Emit the condition */
- compile_condition( parse->left, c, 0, f );
- /* Emit the if block */
- compile_parse( parse->right, c, result_location );
- if ( parse->third->type != PARSE_NULL || result_location != RESULT_NONE )
- {
- /* Emit the else block */
- int end = compile_new_label( c );
- compile_emit_branch( c, INSTR_JUMP, end );
- compile_set_label( c, f );
- compile_parse( parse->third, c, result_location );
- compile_set_label( c, end );
- }
- else
- compile_set_label( c, f );
-
- }
- else if ( parse->type == PARSE_WHILE )
- {
- int nested_result = result_location == RESULT_NONE
- ? RESULT_NONE
- : RESULT_RETURN;
- int test = compile_new_label( c );
- int top = compile_new_label( c );
- /* Make sure that we return an empty list if the loop runs zero times.
- */
- adjust_result( c, RESULT_NONE, nested_result );
- /* Jump to the loop test. */
- compile_emit_branch( c, INSTR_JUMP, test );
- compile_set_label( c, top );
- /* Emit the loop body. */
- compile_parse( parse->right, c, nested_result );
- /* Emit the condition. */
- compile_set_label( c, test );
- compile_condition( parse->left, c, 1, top );
-
- adjust_result( c, nested_result, result_location );
- }
- else if ( parse->type == PARSE_INCLUDE )
- {
- compile_parse( parse->left, c, RESULT_STACK );
- compile_emit( c, INSTR_INCLUDE, 0 );
- compile_emit( c, INSTR_BIND_MODULE_VARIABLES, 0 );
- adjust_result( c, RESULT_NONE, result_location );
- }
- else if ( parse->type == PARSE_MODULE )
- {
- int const nested_result = result_location == RESULT_NONE
- ? RESULT_NONE
- : RESULT_RETURN;
- compile_parse( parse->left, c, RESULT_STACK );
- compile_emit( c, INSTR_PUSH_MODULE, 0 );
- compile_parse( parse->right, c, nested_result );
- compile_emit( c, INSTR_POP_MODULE, 0 );
- adjust_result( c, nested_result, result_location );
- }
- else if ( parse->type == PARSE_CLASS )
- {
- /* Evaluate the class name. */
- compile_parse( parse->left->right, c, RESULT_STACK );
- /* Evaluate the base classes. */
- if ( parse->left->left )
- compile_parse( parse->left->left->right, c, RESULT_STACK );
- else
- compile_emit( c, INSTR_PUSH_EMPTY, 0 );
- compile_emit( c, INSTR_CLASS, 0 );
- compile_parse( parse->right, c, RESULT_NONE );
- compile_emit( c, INSTR_BIND_MODULE_VARIABLES, 0 );
- compile_emit( c, INSTR_POP_MODULE, 0 );
-
- adjust_result( c, RESULT_NONE, result_location );
- }
- else if ( parse->type == PARSE_LIST )
- {
- OBJECT * const o = parse->string;
- char const * s = object_str( o );
- VAR_PARSE_GROUP * group;
- current_file = object_str( parse->file );
- current_line = parse->line;
- group = parse_expansion( &s );
- var_parse_group_compile( group, c );
- var_parse_group_free( group );
- adjust_result( c, RESULT_STACK, result_location );
- }
- else if ( parse->type == PARSE_LOCAL )
- {
- int nested_result = result_location == RESULT_NONE
- ? RESULT_NONE
- : RESULT_RETURN;
- /* This should be left recursive group of compile_appends. */
- PARSE * vars = parse->left;
-
- /* Special case an empty list of vars */
- if ( vars->type == PARSE_NULL )
- {
- compile_parse( parse->right, c, RESULT_NONE );
- compile_parse( parse->third, c, result_location );
- nested_result = result_location;
- }
- /* Check whether there is exactly one variable with a constant name. */
- else if ( vars->left->type == PARSE_NULL &&
- vars->right->type == PARSE_LIST )
- {
- char const * s = object_str( vars->right->string );
- VAR_PARSE_GROUP * group;
- current_file = object_str( parse->file );
- current_line = parse->line;
- group = parse_expansion( &s );
- if ( group->elems->size == 1 && dynamic_array_at( VAR_PARSE *,
- group->elems, 0 )->type == VAR_PARSE_TYPE_STRING )
- {
- int const name = compile_emit_constant( c, (
- (VAR_PARSE_STRING *)dynamic_array_at( VAR_PARSE *,
- group->elems, 0 ) )->s );
- var_parse_group_free( group );
- compile_parse( parse->right, c, RESULT_STACK );
- compile_emit( c, INSTR_PUSH_LOCAL, name );
- compile_parse( parse->third, c, nested_result );
- compile_emit( c, INSTR_POP_LOCAL, name );
- }
- else
- {
- var_parse_group_compile( group, c );
- var_parse_group_free( group );
- compile_parse( parse->right, c, RESULT_STACK );
- compile_emit( c, INSTR_PUSH_LOCAL_GROUP, 0 );
- compile_parse( parse->third, c, nested_result );
- compile_emit( c, INSTR_POP_LOCAL_GROUP, 0 );
- }
- }
- else
- {
- compile_parse( parse->left, c, RESULT_STACK );
- compile_parse( parse->right, c, RESULT_STACK );
- compile_emit( c, INSTR_PUSH_LOCAL_GROUP, 0 );
- compile_parse( parse->third, c, nested_result );
- compile_emit( c, INSTR_POP_LOCAL_GROUP, 0 );
- }
- adjust_result( c, nested_result, result_location );
- }
- else if ( parse->type == PARSE_ON )
- {
- if ( parse->right->type == PARSE_APPEND &&
- parse->right->left->type == PARSE_NULL &&
- parse->right->right->type == PARSE_LIST )
- {
- /* [ on $(target) return $(variable) ] */
- PARSE * value = parse->right->right;
- OBJECT * const o = value->string;
- char const * s = object_str( o );
- VAR_PARSE_GROUP * group;
- OBJECT * varname = 0;
- current_file = object_str( value->file );
- current_line = value->line;
- group = parse_expansion( &s );
- if ( group->elems->size == 1 )
- {
- VAR_PARSE * one = dynamic_array_at( VAR_PARSE *, group->elems, 0 );
- if ( one->type == VAR_PARSE_TYPE_VAR )
- {
- VAR_PARSE_VAR * var = ( VAR_PARSE_VAR * )one;
- if ( var->modifiers->size == 0 && !var->subscript && var->name->elems->size == 1 )
- {
- VAR_PARSE * name = dynamic_array_at( VAR_PARSE *, var->name->elems, 0 );
- if ( name->type == VAR_PARSE_TYPE_STRING )
- {
- varname = ( ( VAR_PARSE_STRING * )name )->s;
- }
- }
- }
- }
- if ( varname )
- {
- /* We have one variable with a fixed name and no modifiers. */
- compile_parse( parse->left, c, RESULT_STACK );
- compile_emit( c, INSTR_GET_ON, compile_emit_constant( c, varname ) );
- }
- else
- {
- /* Too complex. Fall back on push/pop. */
- int end = compile_new_label( c );
- compile_parse( parse->left, c, RESULT_STACK );
- compile_emit_branch( c, INSTR_PUSH_ON, end );
- var_parse_group_compile( group, c );
- compile_emit( c, INSTR_POP_ON, 0 );
- compile_set_label( c, end );
- }
- var_parse_group_free( group );
- }
- else
- {
- int end = compile_new_label( c );
- compile_parse( parse->left, c, RESULT_STACK );
- compile_emit_branch( c, INSTR_PUSH_ON, end );
- compile_parse( parse->right, c, RESULT_STACK );
- compile_emit( c, INSTR_POP_ON, 0 );
- compile_set_label( c, end );
- }
- adjust_result( c, RESULT_STACK, result_location );
- }
- else if ( parse->type == PARSE_RULE )
- {
- PARSE * p;
- int n = 0;
- VAR_PARSE_GROUP * group;
- char const * s = object_str( parse->string );
-
- if ( parse->left->left || parse->left->right->type != PARSE_NULL )
- for ( p = parse->left; p; p = p->left )
- {
- compile_parse( p->right, c, RESULT_STACK );
- ++n;
- }
-
- current_file = object_str( parse->file );
- current_line = parse->line;
- group = parse_expansion( &s );
-
- if ( group->elems->size == 2 &&
- dynamic_array_at( VAR_PARSE *, group->elems, 0 )->type == VAR_PARSE_TYPE_VAR &&
- dynamic_array_at( VAR_PARSE *, group->elems, 1 )->type == VAR_PARSE_TYPE_STRING &&
- ( object_str( ( (VAR_PARSE_STRING *)dynamic_array_at( VAR_PARSE *, group->elems, 1 ) )->s )[ 0 ] == '.' ) )
- {
- VAR_PARSE_STRING * access = (VAR_PARSE_STRING *)dynamic_array_at( VAR_PARSE *, group->elems, 1 );
- OBJECT * member = object_new( object_str( access->s ) + 1 );
- /* Emit the object */
- var_parse_var_compile( (VAR_PARSE_VAR *)dynamic_array_at( VAR_PARSE *, group->elems, 0 ), c );
- var_parse_group_free( group );
- compile_emit( c, INSTR_CALL_MEMBER_RULE, n );
- compile_emit( c, compile_emit_constant( c, member ), parse->line );
- object_free( member );
- }
- else
- {
- var_parse_group_compile( group, c );
- var_parse_group_free( group );
- compile_emit( c, INSTR_CALL_RULE, n );
- compile_emit( c, compile_emit_constant( c, parse->string ), parse->line );
- }
-
- adjust_result( c, RESULT_STACK, result_location );
- }
- else if ( parse->type == PARSE_RULES )
- {
- do compile_parse( parse->left, c, RESULT_NONE );
- while ( ( parse = parse->right )->type == PARSE_RULES );
- compile_parse( parse, c, result_location );
- }
- else if ( parse->type == PARSE_SET )
- {
- PARSE * vars = parse->left;
- unsigned int op_code;
- unsigned int op_code_group;
-
- switch ( parse->num )
- {
- case ASSIGN_APPEND: op_code = INSTR_APPEND; op_code_group = INSTR_APPEND_GROUP; break;
- case ASSIGN_DEFAULT: op_code = INSTR_DEFAULT; op_code_group = INSTR_DEFAULT_GROUP; break;
- default: op_code = INSTR_SET; op_code_group = INSTR_SET_GROUP; break;
- }
-
- /* Check whether there is exactly one variable with a constant name. */
- if ( vars->type == PARSE_LIST )
- {
- char const * s = object_str( vars->string );
- VAR_PARSE_GROUP * group;
- current_file = object_str( parse->file );
- current_line = parse->line;
- group = parse_expansion( &s );
- if ( group->elems->size == 1 && dynamic_array_at( VAR_PARSE *,
- group->elems, 0 )->type == VAR_PARSE_TYPE_STRING )
- {
- int const name = compile_emit_constant( c, (
- (VAR_PARSE_STRING *)dynamic_array_at( VAR_PARSE *,
- group->elems, 0 ) )->s );
- var_parse_group_free( group );
- compile_parse( parse->right, c, RESULT_STACK );
- if ( result_location != RESULT_NONE )
- {
- compile_emit( c, INSTR_SET_RESULT, 1 );
- }
- compile_emit( c, op_code, name );
- }
- else
- {
- var_parse_group_compile( group, c );
- var_parse_group_free( group );
- compile_parse( parse->right, c, RESULT_STACK );
- if ( result_location != RESULT_NONE )
- {
- compile_emit( c, INSTR_SET_RESULT, 1 );
- }
- compile_emit( c, op_code_group, 0 );
- }
- }
- else
- {
- compile_parse( parse->left, c, RESULT_STACK );
- compile_parse( parse->right, c, RESULT_STACK );
- if ( result_location != RESULT_NONE )
- {
- compile_emit( c, INSTR_SET_RESULT, 1 );
- }
- compile_emit( c, op_code_group, 0 );
- }
- if ( result_location != RESULT_NONE )
- {
- adjust_result( c, RESULT_RETURN, result_location );
- }
- }
- else if ( parse->type == PARSE_SETCOMP )
- {
- int n_args;
- struct arg_list * args = arg_list_compile( parse->right, &n_args );
- int const rule_id = compile_emit_rule( c, parse->string, parse->left,
- n_args, args, parse->num );
- compile_emit( c, INSTR_RULE, rule_id );
- adjust_result( c, RESULT_NONE, result_location );
- }
- else if ( parse->type == PARSE_SETEXEC )
- {
- int const actions_id = compile_emit_actions( c, parse );
- compile_parse( parse->left, c, RESULT_STACK );
- compile_emit( c, INSTR_ACTIONS, actions_id );
- adjust_result( c, RESULT_NONE, result_location );
- }
- else if ( parse->type == PARSE_SETTINGS )
- {
- compile_parse( parse->left, c, RESULT_STACK );
- compile_parse( parse->third, c, RESULT_STACK );
- compile_parse( parse->right, c, RESULT_STACK );
-
- switch ( parse->num )
- {
- case ASSIGN_APPEND: compile_emit( c, INSTR_APPEND_ON, 0 ); break;
- case ASSIGN_DEFAULT: compile_emit( c, INSTR_DEFAULT_ON, 0 ); break;
- default: compile_emit( c, INSTR_SET_ON, 0 ); break;
- }
-
- adjust_result( c, RESULT_STACK, result_location );
- }
- else if ( parse->type == PARSE_SWITCH )
- {
- int const switch_end = compile_new_label( c );
- compile_parse( parse->left, c, RESULT_STACK );
-
- for ( parse = parse->right; parse; parse = parse->right )
- {
- int const id = compile_emit_constant( c, parse->left->string );
- int const next_case = compile_new_label( c );
- compile_emit( c, INSTR_PUSH_CONSTANT, id );
- compile_emit_branch( c, INSTR_JUMP_NOT_GLOB, next_case );
- compile_parse( parse->left->left, c, result_location );
- compile_emit_branch( c, INSTR_JUMP, switch_end );
- compile_set_label( c, next_case );
- }
- compile_emit( c, INSTR_POP, 0 );
- adjust_result( c, RESULT_NONE, result_location );
- compile_set_label( c, switch_end );
- }
- else if ( parse->type == PARSE_NULL )
- adjust_result( c, RESULT_NONE, result_location );
- else
- assert( !"unknown PARSE type." );
-}
-
-OBJECT * function_rulename( FUNCTION * function )
-{
- return function->rulename;
-}
-
-void function_set_rulename( FUNCTION * function, OBJECT * rulename )
-{
- function->rulename = rulename;
-}
-
-void function_location( FUNCTION * function_, OBJECT * * file, int * line )
-{
- if ( function_->type == FUNCTION_BUILTIN )
- {
- *file = constant_builtin;
- *line = -1;
- }
-#ifdef HAVE_PYTHON
- if ( function_->type == FUNCTION_PYTHON )
- {
- *file = constant_builtin;
- *line = -1;
- }
-#endif
- else
- {
- JAM_FUNCTION * function = (JAM_FUNCTION *)function_;
- assert( function_->type == FUNCTION_JAM );
- *file = function->file;
- *line = function->line;
- }
-}
-
-static struct arg_list * arg_list_compile_builtin( char const * * args,
- int * num_arguments );
-
-F
<TRUNCATED>
[27/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/compile.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/compile.c b/ext/kenlm/jam-files/engine/compile.c
deleted file mode 100644
index db46937..0000000
--- a/ext/kenlm/jam-files/engine/compile.c
+++ /dev/null
@@ -1,271 +0,0 @@
-/*
- * Copyright 1993, 2000 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * compile.c - compile parsed jam statements
- *
- * External routines:
- * evaluate_rule() - execute a rule invocation
- *
- * Internal routines:
- * debug_compile() - printf with indent to show rule expansion
- */
-
-#include "jam.h"
-#include "compile.h"
-
-#include "builtins.h"
-#include "class.h"
-#include "constants.h"
-#include "hash.h"
-#include "hdrmacro.h"
-#include "make.h"
-#include "modules.h"
-#include "parse.h"
-#include "rules.h"
-#include "search.h"
-#include "strings.h"
-#include "variable.h"
-
-#include <assert.h>
-#include <stdarg.h>
-#include <string.h>
-
-
-static void debug_compile( int which, char const * s, FRAME * );
-
-/* Internal functions from builtins.c */
-void backtrace( FRAME * );
-void backtrace_line( FRAME * );
-void print_source_line( FRAME * );
-void unknown_rule( FRAME *, char const * key, module_t *, OBJECT * rule_name );
-
-
-/*
- * evaluate_rule() - execute a rule invocation
- */
-
-LIST * evaluate_rule( RULE * rule, OBJECT * rulename, FRAME * frame )
-{
- LIST * result = L0;
- profile_frame prof[ 1 ];
- module_t * prev_module = frame->module;
-
- if ( DEBUG_COMPILE )
- {
- /* Try hard to indicate in which module the rule is going to execute. */
- char buf[ 256 ] = "";
- if ( rule->module->name )
- {
- strncat( buf, object_str( rule->module->name ), sizeof( buf ) -
- 1 );
- strncat( buf, ".", sizeof( buf ) - 1 );
- if ( strncmp( buf, object_str( rule->name ), strlen( buf ) ) == 0 )
- {
- buf[ 0 ] = 0;
- }
- }
- strncat( buf, object_str( rule->name ), sizeof( buf ) - 1 );
- debug_compile( 1, buf, frame );
-
- lol_print( frame->args );
- printf( "\n" );
- }
-
- if ( rule->procedure && rule->module != prev_module )
- {
- /* Propagate current module to nested rule invocations. */
- frame->module = rule->module;
- }
-
- /* Record current rule name in frame. */
- if ( rule->procedure )
- {
- frame->rulename = object_str( rulename );
- /* And enter record profile info. */
- if ( DEBUG_PROFILE )
- profile_enter( function_rulename( rule->procedure ), prof );
- }
-
- /* Check traditional targets $(<) and sources $(>). */
- if ( !rule->actions && !rule->procedure )
- unknown_rule( frame, NULL, frame->module, rule->name );
-
- /* If this rule will be executed for updating the targets then construct the
- * action for make().
- */
- if ( rule->actions )
- {
- TARGETS * t;
-
- /* The action is associated with this instance of this rule. */
- ACTION * const action = (ACTION *)BJAM_MALLOC( sizeof( ACTION ) );
- memset( (char *)action, '\0', sizeof( *action ) );
-
- action->rule = rule;
- action->targets = targetlist( (TARGETS *)0, lol_get( frame->args, 0 ) );
- action->sources = targetlist( (TARGETS *)0, lol_get( frame->args, 1 ) );
- action->refs = 1;
-
- /* If we have a group of targets all being built using the same action
- * then we must not allow any of them to be used as sources unless they
- * are all up to date and their action does not need to be run or their
- * action has had a chance to finish its work and build all of them
- * anew.
- *
- * Without this it might be possible, in case of a multi-process build,
- * for their action, triggered to building one of the targets, to still
- * be running when another target in the group reports as done in order
- * to avoid triggering the same action again and gets used prematurely.
- *
- * As a quick-fix to achieve this effect we make all the targets list
- * each other as 'included targets'. More precisely, we mark the first
- * listed target as including all the other targets in the list and vice
- * versa. This makes anyone depending on any of those targets implicitly
- * depend on all of them, thus making sure none of those targets can be
- * used as sources until all of them have been built. Note that direct
- * dependencies could not have been used due to the 'circular
- * dependency' issue.
- *
- * TODO: Although the current implementation solves the problem of one
- * of the targets getting used before its action completes its work, it
- * also forces the action to run whenever any of the targets in the
- * group is not up to date even though some of them might not actually
- * be used by the targets being built. We should see how we can
- * correctly recognize such cases and use that to avoid running the
- * action if possible and not rebuild targets not actually depending on
- * targets that are not up to date.
- *
- * TODO: Current solution using fake INCLUDES relations may cause
- * actions to be run when the affected targets are built by multiple
- * actions. E.g. if we have the following actions registered in the
- * order specified:
- * (I) builds targets A & B
- * (II) builds target B
- * and we want to build a target depending on target A, then both
- * actions (I) & (II) will be run, even though the second one does not
- * have any direct relationship to target A. Consider whether this is
- * desired behaviour or not. It could be that Boost Build should (or
- * possibly already does) run all actions registered for a given target
- * if any of them needs to be run in which case our INCLUDES relations
- * are not actually causing any actions to be run that would not have
- * been run without them.
- */
- if ( action->targets )
- {
- TARGET * const t0 = action->targets->target;
- for ( t = action->targets->next; t; t = t->next )
- {
- target_include( t->target, t0 );
- target_include( t0, t->target );
- }
- }
-
- /* Append this action to the actions of each target. */
- for ( t = action->targets; t; t = t->next )
- t->target->actions = actionlist( t->target->actions, action );
-
- action_free( action );
- }
-
- /* Now recursively compile any parse tree associated with this rule.
- * function_refer()/function_free() call pair added to ensure the rule does
- * not get freed while in use.
- */
- if ( rule->procedure )
- {
- FUNCTION * const function = rule->procedure;
- function_refer( function );
- result = function_run( function, frame, stack_global() );
- function_free( function );
- }
-
- if ( DEBUG_PROFILE && rule->procedure )
- profile_exit( prof );
-
- if ( DEBUG_COMPILE )
- debug_compile( -1, 0, frame );
-
- return result;
-}
-
-
-/*
- * Call the given rule with the specified parameters. The parameters should be
- * of type LIST* and end with a NULL pointer. This differs from 'evaluate_rule'
- * in that frame for the called rule is prepared inside 'call_rule'.
- *
- * This function is useful when a builtin rule (in C) wants to call another rule
- * which might be implemented in Jam.
- */
-
-LIST * call_rule( OBJECT * rulename, FRAME * caller_frame, ... )
-{
- va_list va;
- LIST * result;
-
- FRAME inner[ 1 ];
- frame_init( inner );
- inner->prev = caller_frame;
- inner->prev_user = caller_frame->module->user_module
- ? caller_frame
- : caller_frame->prev_user;
- inner->module = caller_frame->module;
-
- va_start( va, caller_frame );
- for ( ; ; )
- {
- LIST * const l = va_arg( va, LIST * );
- if ( !l )
- break;
- lol_add( inner->args, l );
- }
- va_end( va );
-
- result = evaluate_rule( bindrule( rulename, inner->module ), rulename, inner );
-
- frame_free( inner );
-
- return result;
-}
-
-
-/*
- * debug_compile() - printf with indent to show rule expansion
- */
-
-static void debug_compile( int which, char const * s, FRAME * frame )
-{
- static int level = 0;
- static char indent[ 36 ] = ">>>>|>>>>|>>>>|>>>>|>>>>|>>>>|>>>>|";
-
- if ( which >= 0 )
- {
- int i;
-
- print_source_line( frame );
-
- i = ( level + 1 ) * 2;
- while ( i > 35 )
- {
- fputs( indent, stdout );
- i -= 35;
- }
-
- printf( "%*.*s ", i, i, indent );
- }
-
- if ( s )
- printf( "%s ", s );
-
- level += which;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/compile.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/compile.h b/ext/kenlm/jam-files/engine/compile.h
deleted file mode 100644
index c70f98b..0000000
--- a/ext/kenlm/jam-files/engine/compile.h
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Copyright 1993, 2000 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * compile.h - compile parsed jam statements
- */
-
-#ifndef COMPILE_DWA20011022_H
-#define COMPILE_DWA20011022_H
-
-#include "frames.h"
-#include "lists.h"
-#include "object.h"
-#include "rules.h"
-
-void compile_builtins();
-
-LIST * evaluate_rule( RULE * rule, OBJECT * rulename, FRAME * );
-LIST * call_rule( OBJECT * rulename, FRAME * caller_frame, ... );
-
-/* Flags for compile_set(), etc */
-
-#define ASSIGN_SET 0x00 /* = assign variable */
-#define ASSIGN_APPEND 0x01 /* += append variable */
-#define ASSIGN_DEFAULT 0x02 /* set only if unset */
-
-/* Flags for compile_setexec() */
-
-#define EXEC_UPDATED 0x01 /* executes updated */
-#define EXEC_TOGETHER 0x02 /* executes together */
-#define EXEC_IGNORE 0x04 /* executes ignore */
-#define EXEC_QUIETLY 0x08 /* executes quietly */
-#define EXEC_PIECEMEAL 0x10 /* executes piecemeal */
-#define EXEC_EXISTING 0x20 /* executes existing */
-
-/* Conditions for compile_if() */
-
-#define EXPR_NOT 0 /* ! cond */
-#define EXPR_AND 1 /* cond && cond */
-#define EXPR_OR 2 /* cond || cond */
-#define EXPR_EXISTS 3 /* arg */
-#define EXPR_EQUALS 4 /* arg = arg */
-#define EXPR_NOTEQ 5 /* arg != arg */
-#define EXPR_LESS 6 /* arg < arg */
-#define EXPR_LESSEQ 7 /* arg <= arg */
-#define EXPR_MORE 8 /* arg > arg */
-#define EXPR_MOREEQ 9 /* arg >= arg */
-#define EXPR_IN 10 /* arg in arg */
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/constants.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/constants.c b/ext/kenlm/jam-files/engine/constants.c
deleted file mode 100644
index 891d322..0000000
--- a/ext/kenlm/jam-files/engine/constants.c
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * Copyright 2011 Steven Watanabe
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/*
- * constants.c - constant objects
- *
- * External functions:
- *
- * constants_init() - initialize constants
- * constants_done() - free constants
- *
- */
-
-#include "constants.h"
-
-
-void constants_init( void )
-{
- constant_empty = object_new( "" );
- constant_dot = object_new( "." );
- constant_plus = object_new( "+" );
- constant_star = object_new( "*" );
- constant_question_mark = object_new( "?" );
- constant_ok = object_new( "ok" );
- constant_true = object_new( "true" );
- constant_name = object_new( "__name__" );
- constant_bases = object_new( "__bases__" );
- constant_class = object_new( "__class__" );
- constant_typecheck = object_new( ".typecheck" );
- constant_builtin = object_new( "(builtin)" );
- constant_HCACHEFILE = object_new( "HCACHEFILE" );
- constant_HCACHEMAXAGE = object_new( "HCACHEMAXAGE" );
- constant_HDRSCAN = object_new( "HDRSCAN" );
- constant_HDRRULE = object_new( "HDRRULE" );
- constant_BINDRULE = object_new( "BINDRULE" );
- constant_LOCATE = object_new( "LOCATE" );
- constant_SEARCH = object_new( "SEARCH" );
- constant_JAM_SEMAPHORE = object_new( "JAM_SEMAPHORE" );
- constant_TIMING_RULE = object_new( "__TIMING_RULE__" );
- constant_ACTION_RULE = object_new( "__ACTION_RULE__" );
- constant_JAMSHELL = object_new( "JAMSHELL" );
- constant_TMPDIR = object_new( "TMPDIR" );
- constant_TMPNAME = object_new( "TMPNAME" );
- constant_TMPFILE = object_new( "TMPFILE" );
- constant_STDOUT = object_new( "STDOUT" );
- constant_STDERR = object_new( "STDERR" );
- constant_JAMDATE = object_new( "JAMDATE" );
- constant_JAM_TIMESTAMP_RESOLUTION = object_new( "JAM_TIMESTAMP_RESOLUTION" );
- constant_JAM_VERSION = object_new( "JAM_VERSION" );
- constant_JAMUNAME = object_new( "JAMUNAME" );
- constant_ENVIRON = object_new( ".ENVIRON" );
- constant_ARGV = object_new( "ARGV" );
- constant_all = object_new( "all" );
- constant_PARALLELISM = object_new( "PARALLELISM" );
- constant_KEEP_GOING = object_new( "KEEP_GOING" );
- constant_other = object_new( "[OTHER]" );
- constant_total = object_new( "[TOTAL]" );
- constant_FILE_DIRSCAN = object_new( "FILE_DIRSCAN" );
- constant_MAIN = object_new( "MAIN" );
- constant_MAIN_MAKE = object_new( "MAIN_MAKE" );
- constant_MAKE_MAKE0 = object_new( "MAKE_MAKE0" );
- constant_MAKE_MAKE1 = object_new( "MAKE_MAKE1" );
- constant_MAKE_MAKE0SORT = object_new( "MAKE_MAKE0SORT" );
- constant_BINDMODULE = object_new( "BINDMODULE" );
- constant_IMPORT_MODULE = object_new( "IMPORT_MODULE" );
- constant_BUILTIN_GLOB_BACK = object_new( "BUILTIN_GLOB_BACK" );
- constant_timestamp = object_new( "timestamp" );
- constant_python = object_new("__python__");
- constant_python_interface = object_new( "python_interface" );
- constant_extra_pythonpath = object_new( "EXTRA_PYTHONPATH" );
- constant_MAIN_PYTHON = object_new( "MAIN_PYTHON" );
-}
-
-void constants_done( void )
-{
- object_free( constant_empty );
- object_free( constant_dot );
- object_free( constant_plus );
- object_free( constant_star );
- object_free( constant_question_mark );
- object_free( constant_ok );
- object_free( constant_true );
- object_free( constant_name );
- object_free( constant_bases );
- object_free( constant_class );
- object_free( constant_typecheck );
- object_free( constant_builtin );
- object_free( constant_HCACHEFILE );
- object_free( constant_HCACHEMAXAGE );
- object_free( constant_HDRSCAN );
- object_free( constant_HDRRULE );
- object_free( constant_BINDRULE );
- object_free( constant_LOCATE );
- object_free( constant_SEARCH );
- object_free( constant_JAM_SEMAPHORE );
- object_free( constant_TIMING_RULE );
- object_free( constant_ACTION_RULE );
- object_free( constant_JAMSHELL );
- object_free( constant_TMPDIR );
- object_free( constant_TMPNAME );
- object_free( constant_TMPFILE );
- object_free( constant_STDOUT );
- object_free( constant_STDERR );
- object_free( constant_JAMDATE );
- object_free( constant_JAM_TIMESTAMP_RESOLUTION );
- object_free( constant_JAM_VERSION );
- object_free( constant_JAMUNAME );
- object_free( constant_ENVIRON );
- object_free( constant_ARGV );
- object_free( constant_all );
- object_free( constant_PARALLELISM );
- object_free( constant_KEEP_GOING );
- object_free( constant_other );
- object_free( constant_total );
- object_free( constant_FILE_DIRSCAN );
- object_free( constant_MAIN );
- object_free( constant_MAIN_MAKE );
- object_free( constant_MAKE_MAKE0 );
- object_free( constant_MAKE_MAKE1 );
- object_free( constant_MAKE_MAKE0SORT );
- object_free( constant_BINDMODULE );
- object_free( constant_IMPORT_MODULE );
- object_free( constant_BUILTIN_GLOB_BACK );
- object_free( constant_timestamp );
- object_free( constant_python );
- object_free( constant_python_interface );
- object_free( constant_extra_pythonpath );
- object_free( constant_MAIN_PYTHON );
-}
-
-OBJECT * constant_empty;
-OBJECT * constant_dot;
-OBJECT * constant_plus;
-OBJECT * constant_star;
-OBJECT * constant_question_mark;
-OBJECT * constant_ok;
-OBJECT * constant_true;
-OBJECT * constant_name;
-OBJECT * constant_bases;
-OBJECT * constant_class;
-OBJECT * constant_typecheck;
-OBJECT * constant_builtin;
-OBJECT * constant_HCACHEFILE;
-OBJECT * constant_HCACHEMAXAGE;
-OBJECT * constant_HDRSCAN;
-OBJECT * constant_HDRRULE;
-OBJECT * constant_BINDRULE;
-OBJECT * constant_LOCATE;
-OBJECT * constant_SEARCH;
-OBJECT * constant_JAM_SEMAPHORE;
-OBJECT * constant_TIMING_RULE;
-OBJECT * constant_ACTION_RULE;
-OBJECT * constant_JAMSHELL;
-OBJECT * constant_TMPDIR;
-OBJECT * constant_TMPNAME;
-OBJECT * constant_TMPFILE;
-OBJECT * constant_STDOUT;
-OBJECT * constant_STDERR;
-OBJECT * constant_JAMDATE;
-OBJECT * constant_JAM_VERSION;
-OBJECT * constant_JAMUNAME;
-OBJECT * constant_ENVIRON;
-OBJECT * constant_ARGV;
-OBJECT * constant_all;
-OBJECT * constant_PARALLELISM;
-OBJECT * constant_KEEP_GOING;
-OBJECT * constant_other;
-OBJECT * constant_total;
-OBJECT * constant_FILE_DIRSCAN;
-OBJECT * constant_MAIN;
-OBJECT * constant_MAIN_MAKE;
-OBJECT * constant_MAKE_MAKE0;
-OBJECT * constant_MAKE_MAKE1;
-OBJECT * constant_MAKE_MAKE0SORT;
-OBJECT * constant_BINDMODULE;
-OBJECT * constant_IMPORT_MODULE;
-OBJECT * constant_BUILTIN_GLOB_BACK;
-OBJECT * constant_timestamp;
-OBJECT * constant_JAM_TIMESTAMP_RESOLUTION;
-OBJECT * constant_python;
-OBJECT * constant_python_interface;
-OBJECT * constant_extra_pythonpath;
-OBJECT * constant_MAIN_PYTHON;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/constants.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/constants.h b/ext/kenlm/jam-files/engine/constants.h
deleted file mode 100644
index 60d7073..0000000
--- a/ext/kenlm/jam-files/engine/constants.h
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Copyright 2011 Steven Watanabe
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/*
- * constants.h - constant objects
- */
-
-#ifndef BOOST_JAM_CONSTANTS_H
-#define BOOST_JAM_CONSTANTS_H
-
-#include "object.h"
-
-void constants_init( void );
-void constants_done( void );
-
-extern OBJECT * constant_empty; /* "" */
-extern OBJECT * constant_dot; /* "." */
-extern OBJECT * constant_plus; /* "+" */
-extern OBJECT * constant_star; /* "*" */
-extern OBJECT * constant_question_mark; /* "?" */
-extern OBJECT * constant_ok; /* "ok" */
-extern OBJECT * constant_true; /* "true" */
-extern OBJECT * constant_name; /* "__name__" */
-extern OBJECT * constant_bases; /* "__bases__" */
-extern OBJECT * constant_class; /* "__class__" */
-extern OBJECT * constant_typecheck; /* ".typecheck" */
-extern OBJECT * constant_builtin; /* "(builtin)" */
-extern OBJECT * constant_HCACHEFILE; /* "HCACHEFILE" */
-extern OBJECT * constant_HCACHEMAXAGE; /* "HCACHEMAXAGE" */
-extern OBJECT * constant_HDRSCAN; /* "HDRSCAN" */
-extern OBJECT * constant_HDRRULE; /* "HDRRULE" */
-extern OBJECT * constant_BINDRULE; /* "BINDRULE" */
-extern OBJECT * constant_LOCATE; /* "LOCATE" */
-extern OBJECT * constant_SEARCH; /* "SEARCH" */
-extern OBJECT * constant_JAM_SEMAPHORE; /* "JAM_SEMAPHORE" */
-extern OBJECT * constant_TIMING_RULE; /* "__TIMING_RULE__" */
-extern OBJECT * constant_ACTION_RULE; /* "__ACTION_RULE__" */
-extern OBJECT * constant_JAMSHELL; /* "JAMSHELL" */
-extern OBJECT * constant_TMPDIR; /* "TMPDIR" */
-extern OBJECT * constant_TMPNAME; /* "TMPNAME" */
-extern OBJECT * constant_TMPFILE; /* "TMPFILE" */
-extern OBJECT * constant_STDOUT; /* "STDOUT" */
-extern OBJECT * constant_STDERR; /* "STDERR" */
-extern OBJECT * constant_JAMDATE; /* "JAMDATE" */
-extern OBJECT * constant_JAM_TIMESTAMP_RESOLUTION; /* "JAM_TIMESTAMP_RESOLUTION" */
-extern OBJECT * constant_JAM_VERSION; /* "JAM_VERSION" */
-extern OBJECT * constant_JAMUNAME; /* "JAMUNAME" */
-extern OBJECT * constant_ENVIRON; /* ".ENVIRON" */
-extern OBJECT * constant_ARGV; /* "ARGV" */
-extern OBJECT * constant_all; /* "all" */
-extern OBJECT * constant_PARALLELISM; /* "PARALLELISM" */
-extern OBJECT * constant_KEEP_GOING; /* "KEEP_GOING" */
-extern OBJECT * constant_other; /* "[OTHER]" */
-extern OBJECT * constant_total; /* "[TOTAL]" */
-extern OBJECT * constant_FILE_DIRSCAN; /* "FILE_DIRSCAN" */
-extern OBJECT * constant_MAIN; /* "MAIN" */
-extern OBJECT * constant_MAIN_MAKE; /* "MAIN_MAKE" */
-extern OBJECT * constant_MAKE_MAKE0; /* "MAKE_MAKE0" */
-extern OBJECT * constant_MAKE_MAKE1; /* "MAKE_MAKE1" */
-extern OBJECT * constant_MAKE_MAKE0SORT; /* "MAKE_MAKE0SORT" */
-extern OBJECT * constant_BINDMODULE; /* "BINDMODULE" */
-extern OBJECT * constant_IMPORT_MODULE; /* "IMPORT_MODULE" */
-extern OBJECT * constant_BUILTIN_GLOB_BACK; /* "BUILTIN_GLOB_BACK" */
-extern OBJECT * constant_timestamp; /* "timestamp" */
-extern OBJECT * constant_python; /* "__python__" */
-extern OBJECT * constant_python_interface; /* "python_interface" */
-extern OBJECT * constant_extra_pythonpath; /* "EXTRA_PYTHONPATH" */
-extern OBJECT * constant_MAIN_PYTHON; /* "MAIN_PYTHON" */
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/cwd.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/cwd.c b/ext/kenlm/jam-files/engine/cwd.c
deleted file mode 100644
index 7ebe970..0000000
--- a/ext/kenlm/jam-files/engine/cwd.c
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Copyright 2002. Vladimir Prus
- * Copyright 2005. Rene Rivera
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#include "cwd.h"
-
-#include "jam.h"
-#include "mem.h"
-#include "pathsys.h"
-
-#include <assert.h>
-#include <errno.h>
-#include <limits.h>
-
-/* MinGW on Windows declares PATH_MAX in limits.h */
-#if defined( NT ) && !defined( __GNUC__ )
-# include <direct.h>
-# define PATH_MAX _MAX_PATH
-#else
-# include <unistd.h>
-# if defined( __COMO__ )
-# include <linux/limits.h>
-# endif
-#endif
-
-#ifndef PATH_MAX
-# define PATH_MAX 1024
-#endif
-
-
-static OBJECT * cwd_;
-
-
-void cwd_init( void )
-{
- int buffer_size = PATH_MAX;
- char * cwd_buffer = 0;
- int error;
-
- assert( !cwd_ );
-
- do
- {
- char * const buffer = BJAM_MALLOC_RAW( buffer_size );
- cwd_buffer = getcwd( buffer, buffer_size );
- error = errno;
- if ( cwd_buffer )
- {
- /* We store the path using its canonical/long/key format. */
- OBJECT * const cwd = object_new( cwd_buffer );
- cwd_ = path_as_key( cwd );
- object_free( cwd );
- }
- buffer_size *= 2;
- BJAM_FREE_RAW( buffer );
- }
- while ( !cwd_ && error == ERANGE );
-
- if ( !cwd_ )
- {
- perror( "can not get current working directory" );
- exit( EXITBAD );
- }
-}
-
-
-OBJECT * cwd( void )
-{
- assert( cwd_ );
- return cwd_;
-}
-
-
-void cwd_done( void )
-{
- assert( cwd_ );
- object_free( cwd_ );
- cwd_ = NULL;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/cwd.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/cwd.h b/ext/kenlm/jam-files/engine/cwd.h
deleted file mode 100644
index 886714a..0000000
--- a/ext/kenlm/jam-files/engine/cwd.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright 2002. Vladimir Prus
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * cwd.h - manages the current working folder information
- */
-
-#ifndef CWD_H
-#define CWD_H
-
-#include "object.h"
-
-
-/* cwd() - returns the current working folder */
-OBJECT * cwd( void );
-
-/* cwd_init() - initialize the cwd module functionality
- *
- * The current working folder can not change in Boost Jam so this function
- * gets the current working folder information from the OS and stores it
- * internally.
- *
- * Expected to be called at program startup before the program's current
- * working folder has been changed
- */
-void cwd_init( void );
-
-/* cwd_done() - cleans up the cwd module functionality */
-void cwd_done( void );
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/debian/changelog
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/debian/changelog b/ext/kenlm/jam-files/engine/debian/changelog
deleted file mode 100644
index 2908428..0000000
--- a/ext/kenlm/jam-files/engine/debian/changelog
+++ /dev/null
@@ -1,72 +0,0 @@
-bjam (3.1.12-1) unstable; urgency=low
-
- * New upstream release.
-
- -- Rene Rivera <gr...@redshift-software.com> Sat, 01 Oct 2005 00:00:00 +0000
-
-bjam (3.1.11-1) unstable; urgency=low
-
- * New upstream release.
-
- -- Rene Rivera <gr...@redshift-software.com> Sat, 30 Apr 2005 00:00:00 +0000
-
-bjam (3.1.10-1) unstable; urgency=low
-
- * New upstream release.
-
- -- Rene Rivera <gr...@redshift-software.com> Tue, 1 Jun 2004 05:42:35 +0000
-
-bjam (3.1.9-2) unstable; urgency=low
-
- * Use default value of BOOST_BUILD_PATH is not is set in environment.
-
- -- Vladimir Prus <gh...@zigzag.lvk.cs.msu.su> Wed, 17 Dec 2003 16:44:35 +0300
-
-bjam (3.1.9-1) unstable; urgency=low
-
- * Implement NATIVE_FILE builtin and several native rules.
-
- -- Vladimir Prus <gh...@zigzag.lvk.cs.msu.su> Thu, 11 Dec 2003 13:15:26 +0300
-
-bjam (3.1.8-1) unstable; urgency=low
-
- * New upstream release.
-
- -- Vladimir Prus <gh...@zigzag.lvk.cs.msu.su> Tue, 4 Nov 2003 20:50:43 +0300
-
-bjam (3.1.7-1) unstable; urgency=low
-
- * New upstream release.
-
- -- Vladimir Prus <gh...@zigzag.lvk.cs.msu.su> Thu, 11 Sep 2003 10:45:44 +0400
-
-bjam (3.1.6-1) unstable; urgency=low
-
- * New upstream release.
-
- -- Vladimir Prus <gh...@zigzag.lvk.cs.msu.su> Tue, 1 Jul 2003 09:12:18 +0400
-
-bjam (3.1.5-1) unstable; urgency=low
-
- * New upstream release.
-
- -- Vladimir Prus <gh...@zigzag.lvk.cs.msu.su> Mon, 19 May 2003 14:05:13 +0400
-
-bjam (3.1.3-2) unstable; urgency=low
-
- * Changed Debian package to be similar to Jam's package.
-
- -- Vladimir Prus <gh...@cs.msu.su> Thu, 10 Oct 2002 18:43:26 +0400
-
-bjam (3.1.3-1) unstable; urgency=low
-
- * New upstream release.
-
- -- Vladimir Prus <gh...@zigzag.lvk.cs.msu.su> Fri, 4 Oct 2002 18:16:54 +0400
-
-bjam (3.1.2-1) unstable; urgency=low
-
- * Initial Release.
-
- -- Vladimir Prus <gh...@cs.msu.su> Wed, 14 Aug 2002 14:08:00 +0400
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/debian/control
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/debian/control b/ext/kenlm/jam-files/engine/debian/control
deleted file mode 100644
index c7f1519..0000000
--- a/ext/kenlm/jam-files/engine/debian/control
+++ /dev/null
@@ -1,16 +0,0 @@
-Source: bjam
-Section: devel
-Priority: optional
-Maintainer: Vladimir Prus <gh...@cs.msu.su>
-Build-Depends: debhelper (>> 3.0.0), docbook-to-man, bison
-Standards-Version: 3.5.2
-
-Package: bjam
-Architecture: any
-Depends: ${shlibs:Depends}
-Description: Build tool
- Boost.Jam is a portable build tool with its own interpreted language, which
- allows to implement rather complex logic in a readable way and without
- resorting to external programs. It is a descendant of Jam/MR tool modified to
- suit the needs of Boost.Build. In particular, modules and rule parameters
- were added, as well as several new builtins.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/debian/copyright
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/debian/copyright b/ext/kenlm/jam-files/engine/debian/copyright
deleted file mode 100644
index f72e4e3..0000000
--- a/ext/kenlm/jam-files/engine/debian/copyright
+++ /dev/null
@@ -1,25 +0,0 @@
-This package was debianized by Vladimir Prus <gh...@cs.msu.su> on
-Wed, 17 July 2002, 19:27:00 +0400.
-
-Copyright:
-
- /+\
- +\ Copyright 1993-2002 Christopher Seiwald and Perforce Software, Inc.
- \+/
-
- This is Release 2.4 of Jam/MR, a make-like program.
-
- License is hereby granted to use this software and distribute it
- freely, as long as this copyright notice is retained and modifications
- are clearly marked.
-
- ALL WARRANTIES ARE HEREBY DISCLAIMED.
-
-Some portions are also:
-
- Copyright 2001-2006 David Abrahams.
- Copyright 2002-2006 Rene Rivera.
- Copyright 2003-2006 Vladimir Prus.
-
- Distributed under the Boost Software License, Version 1.0.
- (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/debian/jam.man.sgml
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/debian/jam.man.sgml b/ext/kenlm/jam-files/engine/debian/jam.man.sgml
deleted file mode 100644
index ee21d4d..0000000
--- a/ext/kenlm/jam-files/engine/debian/jam.man.sgml
+++ /dev/null
@@ -1,236 +0,0 @@
-<!doctype refentry PUBLIC "-//OASIS//DTD DocBook V4.1//EN" [
-
-<!-- Process this file with docbook-to-man to generate an nroff manual
- page: `docbook-to-man manpage.sgml > manpage.1'. You may view
- the manual page with: `docbook-to-man manpage.sgml | nroff -man |
- less'. A typical entry in a Makefile or Makefile.am is:
-
-manpage.1: manpage.sgml
- docbook-to-man $< > $@
- -->
-
- <!ENTITY dhfirstname "<firstname>Yann</firstname>">
- <!ENTITY dhsurname "<surname>Dirson</surname>">
- <!-- Please adjust the date whenever revising the manpage. -->
- <!ENTITY dhdate "<date>mai 23, 2001</date>">
- <!ENTITY dhemail "<email>dirson@debian.org</email>">
- <!ENTITY dhusername "Yann Dirson">
- <!ENTITY dhpackage "jam">
-
- <!ENTITY debian "<productname>Debian GNU/Linux</productname>">
- <!ENTITY gnu "<acronym>GNU</acronym>">
-]>
-
-<refentry>
- <refentryinfo>
- <address>
- &dhemail;
- </address>
- <author>
- &dhfirstname;
- &dhsurname;
- </author>
- <copyright>
- <year>2001</year>
- <holder>&dhusername;</holder>
- </copyright>
- &dhdate;
- </refentryinfo>
-
- <refmeta>
- <refentrytitle>JAM</refentrytitle>
- <manvolnum>1</manvolnum>
- </refmeta>
-
- <refnamediv>
- <refname>Jam/MR</refname>
- <refpurpose>Make(1) Redux</refpurpose>
- </refnamediv>
-
- <refsynopsisdiv>
- <cmdsynopsis>
- <command>jam</command>
-
- <arg><option>-a</option></arg>
- <arg><option>-n</option></arg>
- <arg><option>-v</option></arg>
-
- <arg><option>-d <replaceable/debug/</option></arg>
- <arg><option>-f <replaceable/jambase/</option></arg>
- <arg><option>-j <replaceable/jobs/</option></arg>
- <arg><option>-o <replaceable/actionsfile/</option></arg>
- <arg><option>-s <replaceable/var/=<replaceable/value/</option></arg>
- <arg><option>-t <replaceable/target/</option></arg>
-
- <arg repeat><option><replaceable/target/</option></arg>
- </cmdsynopsis>
- </refsynopsisdiv>
-
- <refsect1>
- <title>DESCRIPTION</title>
-
- <para>Jam is a program construction tool, like make(1).</para>
-
- <para>Jam recursively builds target files from source files, using
- dependency information and updating actions expressed in the
- Jambase file, which is written in jam's own interpreted language.
- The default Jambase is compiled into jam and provides a
- boilerplate for common use, relying on a user-provide file
- "Jamfile" to enumerate actual targets and sources.</para>
- </refsect1>
-
- <refsect1>
- <title>OPTIONS</title>
-
- <variablelist>
- <varlistentry>
- <term><option/-a/</term>
- <listitem>
- <para>Build all targets anyway, even if they are up-to-date.</para>
- </listitem>
- </varlistentry>
-
- <varlistentry>
- <term><option>-d <replaceable/n/</option></term>
- <listitem>
- <para>Enable cummulative debugging levels from 1 to
- <replaceable/n/. Interesting values are:
-
- <glosslist>
- <glossentry><glossterm/1/ <glossdef><simpara/Show
- actions (the default)/</glossdef></glossentry>
-
- <glossentry><glossterm/2/ <glossdef><simpara/Show
- "quiet" actions and display all action
- text/</glossdef></glossentry>
-
- <glossentry><glossterm/3/ <glossdef><simpara>Show
- dependency analysis, and target/source
- timestamps/paths</simpara></glossdef></glossentry>
-
- <glossentry><glossterm/4/ <glossdef><simpara/Show shell
- arguments/</glossdef></glossentry>
-
- <glossentry><glossterm/5/ <glossdef><simpara/Show rule
- invocations and variable
- expansions/</glossdef></glossentry>
-
- <glossentry><glossterm/6/ <glossdef><simpara>Show
- directory/header file/archive
- scans</simpara></glossdef></glossentry>
-
- <glossentry><glossterm/7/ <glossdef><simpara/Show
- variable settings/</glossdef></glossentry>
-
- <glossentry><glossterm/8/ <glossdef><simpara/Show
- variable fetches/</glossdef></glossentry>
-
- <glossentry><glossterm/9/ <glossdef><simpara/Show
- variable manipulation, scanner
- tokens/</glossdef></glossentry>
- </glosslist>
- </para>
- </listitem>
- </varlistentry>
-
- <varlistentry>
- <term><option>-d +<replaceable/n/</option></term>
- <listitem>
- <para>Enable debugging level <replaceable/n/.</para>
- </listitem>
- </varlistentry>
-
- <varlistentry>
- <term><option/-d 0/</term>
- <listitem>
- <para>Turn off all debugging levels. Only errors are not
- suppressed.</para>
- </listitem>
- </varlistentry>
-
- <varlistentry>
- <term><option>-f <replaceable/jambase/</option></term>
- <listitem>
- <para>Read <replaceable/jambase/ instead of using the
- built-in Jambase. Only one <option/-f/ flag is permitted,
- but the <replaceable/jambase/ may explicitly include other
- files.</para>
- </listitem>
- </varlistentry>
-
- <varlistentry>
- <term><option>-j <replaceable/n/</option></term>
- <listitem>
- <para>Run up to <replaceable/n/ shell commands concurrently
- (UNIX and NT only). The default is 1.</para>
- </listitem>
- </varlistentry>
-
- <varlistentry>
- <term><option/-n/</term>
- <listitem>
- <para>Don't actually execute the updating actions, but do
- everything else. This changes the debug level default to
- <option/-d2/.</para>
- </listitem>
- </varlistentry>
-
- <varlistentry>
- <term><option>-o <replaceable/file/</option></term>
- <listitem>
- <para>Write the updating actions to the specified file
- instead of running them (or outputting them, as on the
- Mac).</para>
- </listitem>
- </varlistentry>
-
- <varlistentry>
- <term><option>-s <replaceable/var/=<replaceable/value/</option></term>
- <listitem>
- <para>Set the variable <replaceable/var/ to
- <replaceable/value/, overriding both internal variables and
- variables imported from the environment. </para>
- </listitem>
- </varlistentry>
-
- <varlistentry>
- <term><option>-t <replaceable/target/</option></term>
- <listitem>
- <para>Rebuild <replaceable/target/ and everything that
- depends on it, even if it is up-to-date.</para>
- </listitem>
- </varlistentry>
-
- <varlistentry>
- <term><option/-v/</term>
- <listitem>
- <para>Print the version of jam and exit.</para>
- </listitem>
- </varlistentry>
-
- </variablelist>
- </refsect1>
-
- <refsect1>
- <title>SEE ALSO</title>
-
- <para>Jam is documented fully in HTML pages available on Debian
- systems from
- <filename>/usr/share/doc/jam/Jam.html</filename>.</para>
- </refsect1>
-
- <refsect1>
- <title>AUTHOR</title>
-
- <para>This manual page was created by &dhusername; &dhemail; from
- the <filename/Jam.html/ documentation, for the &debian; system
- (but may be used by others).</para>
- </refsect1>
-</refentry>
-
-<!-- Keep this comment at the end of the file
-Local variables:
-sgml-omittag:t
-sgml-shorttag:t
-End:
--->
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/debian/rules
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/debian/rules b/ext/kenlm/jam-files/engine/debian/rules
deleted file mode 100755
index 756052a..0000000
--- a/ext/kenlm/jam-files/engine/debian/rules
+++ /dev/null
@@ -1,73 +0,0 @@
-#!/usr/bin/make -f
-# Sample debian/rules that uses debhelper.
-# GNU copyright 1997 to 1999 by Joey Hess.
-# GNU copyright 2001 by Yann Dirson.
-
-# This is the debian/rules file for packages jam and ftjam
-# It should be usable with both packages without any change
-
-# Uncomment this to turn on verbose mode.
-#export DH_VERBOSE=1
-
-# This is the debhelper compatability version to use.
-export DH_COMPAT=3
-
-topdir=$(shell pwd)
-
-jam=bjam
-binname=bjam
-
-build: build-stamp
-build-stamp: debian/jam.1
- dh_testdir
-
- ./build.sh
-
- touch build-stamp
-
-%.1: %.man.sgml
- /usr/bin/docbook-to-man $< > $@
-
-clean:
- dh_testdir
- dh_testroot
- rm -f build-stamp
- rm -rf bin.*
- rm -f jam0 debian/jam.1
- dh_clean
-
-install: build
- dh_testdir
- dh_testroot
- dh_clean -k
- dh_installdirs
-
- install -d ${topdir}/debian/${jam}/usr/bin
- install -m755 bin.linuxx86/bjam ${topdir}/debian/${jam}/usr/bin/
- install -d ${topdir}/debian/${jam}/usr/share/man/man1/
- install -m644 debian/jam.1 ${topdir}/debian/${jam}/usr/share/man/man1/${binname}.1
-
-
-# Build architecture-independent files here.
-binary-indep: build install
-# We have nothing to do by default.
-
-# Build architecture-dependent files here.
-binary-arch: build install
- dh_testdir
- dh_testroot
- dh_installdocs README RELNOTES Jambase *.html
-# dh_installemacsen
-# dh_undocumented
- dh_installchangelogs
- dh_strip
- dh_compress
- dh_fixperms
- dh_installdeb
- dh_shlibdeps
- dh_gencontrol
- dh_md5sums
- dh_builddeb
-
-binary: binary-indep binary-arch
-.PHONY: build clean binary-indep binary-arch binary install configure
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/debug.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/debug.c b/ext/kenlm/jam-files/engine/debug.c
deleted file mode 100644
index 2a65655..0000000
--- a/ext/kenlm/jam-files/engine/debug.c
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Copyright 2005. Rene Rivera
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#include "jam.h"
-#include "debug.h"
-
-#include "hash.h"
-
-
-static profile_frame * profile_stack = 0;
-static struct hash * profile_hash = 0;
-static profile_info profile_other = { 0 };
-static profile_info profile_total = { 0 };
-
-
-profile_frame * profile_init( OBJECT * rulename, profile_frame * frame )
-{
- if ( DEBUG_PROFILE ) profile_enter( rulename, frame );
- return frame;
-}
-
-
-void profile_enter( OBJECT * rulename, profile_frame * frame )
-{
- if ( DEBUG_PROFILE )
- {
- clock_t start = clock();
- profile_info * p;
-
- if ( !profile_hash && rulename )
- profile_hash = hashinit( sizeof( profile_info ), "profile" );
-
- if ( rulename )
- {
- int found;
- p = (profile_info *)hash_insert( profile_hash, rulename, &found );
- if ( !found )
- {
- p->name = rulename;
- p->cumulative = 0;
- p->net = 0;
- p->num_entries = 0;
- p->stack_count = 0;
- p->memory = 0;
- }
- }
- else
- {
- p = &profile_other;
- }
-
- ++p->num_entries;
- ++p->stack_count;
-
- frame->info = p;
-
- frame->caller = profile_stack;
- profile_stack = frame;
-
- frame->entry_time = clock();
- frame->overhead = 0;
- frame->subrules = 0;
-
- /* caller pays for the time it takes to play with the hash table */
- if ( frame->caller )
- frame->caller->overhead += frame->entry_time - start;
- }
-}
-
-
-void profile_memory( long mem )
-{
- if ( DEBUG_PROFILE )
- if ( profile_stack && profile_stack->info )
- profile_stack->info->memory += mem;
-}
-
-
-void profile_exit( profile_frame * frame )
-{
- if ( DEBUG_PROFILE )
- {
- /* Cumulative time for this call. */
- clock_t const t = clock() - frame->entry_time - frame->overhead;
- /* If this rule is already present on the stack, do not add the time for
- * this instance.
- */
- if ( frame->info->stack_count == 1 )
- frame->info->cumulative += t;
- /* Net time does not depend on presense of the same rule in call stack.
- */
- frame->info->net += t - frame->subrules;
-
- if ( frame->caller )
- {
- /* Caller's cumulative time must account for this overhead. */
- frame->caller->overhead += frame->overhead;
- frame->caller->subrules += t;
- }
- /* Pop this stack frame. */
- --frame->info->stack_count;
- profile_stack = frame->caller;
- }
-}
-
-
-static void dump_profile_entry( void * p_, void * ignored )
-{
- profile_info * p = (profile_info *)p_;
- unsigned long mem_each = ( p->memory / ( p->num_entries ? p->num_entries : 1
- ) );
- double cumulative = p->cumulative;
- double net = p->net;
- double q = p->net;
- q /= ( p->num_entries ? p->num_entries : 1 );
- cumulative /= CLOCKS_PER_SEC;
- net /= CLOCKS_PER_SEC;
- q /= CLOCKS_PER_SEC;
- if ( !ignored )
- {
- profile_total.cumulative += p->net;
- profile_total.memory += p->memory;
- }
- printf( "%10ld %12.6f %12.6f %12.8f %10ld %10ld %s\n", p->num_entries,
- cumulative, net, q, p->memory, mem_each, object_str( p->name ) );
-}
-
-
-void profile_dump()
-{
- if ( profile_hash )
- {
- printf( "%10s %12s %12s %12s %10s %10s %s\n", "--count--", "--gross--",
- "--net--", "--each--", "--mem--", "--each--", "--name--" );
- hashenumerate( profile_hash, dump_profile_entry, 0 );
- profile_other.name = constant_other;
- dump_profile_entry( &profile_other, 0 );
- profile_total.name = constant_total;
- dump_profile_entry( &profile_total, (void *)1 );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/debug.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/debug.h b/ext/kenlm/jam-files/engine/debug.h
deleted file mode 100644
index 4151d27..0000000
--- a/ext/kenlm/jam-files/engine/debug.h
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Copyright 2005. Rene Rivera
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#ifndef BJAM_DEBUG_H
-#define BJAM_DEBUG_H
-
-#include "constants.h"
-#include "object.h"
-#include <time.h>
-
-
-typedef struct profile_info
-{
- /* name of rule being called */
- OBJECT * name;
- /* cumulative time spent in rule */
- clock_t cumulative;
- /* time spent in rule proper */
- clock_t net;
- /* number of time rule was entered */
- unsigned long num_entries;
- /* number of the times this function is present in stack */
- unsigned long stack_count;
- /* bytes of memory allocated by the call */
- unsigned long memory;
-} profile_info;
-
-typedef struct profile_frame
-{
- /* permanent storage where data accumulates */
- profile_info * info;
- /* overhead for profiling in this call */
- clock_t overhead;
- /* time of last entry to rule */
- clock_t entry_time;
- /* stack frame of caller */
- struct profile_frame * caller;
- /* time spent in subrules */
- clock_t subrules;
-} profile_frame;
-
-profile_frame * profile_init( OBJECT * rulename, profile_frame * );
-void profile_enter( OBJECT * rulename, profile_frame * );
-void profile_memory( long mem );
-void profile_exit( profile_frame * );
-void profile_dump();
-
-#define PROFILE_ENTER( scope ) profile_frame PROF_ ## scope, *PROF_ ## scope ## _p = profile_init( constant_ ## scope, &PROF_ ## scope )
-#define PROFILE_EXIT( scope ) profile_exit( PROF_ ## scope ## _p )
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/execcmd.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/execcmd.c b/ext/kenlm/jam-files/engine/execcmd.c
deleted file mode 100644
index f751cbf..0000000
--- a/ext/kenlm/jam-files/engine/execcmd.c
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Copyright 1993, 1995 Christopher Seiwald.
- * Copyright 2007 Noel Belcourt.
- *
- * Utility functions shared between different exec*.c platform specific
- * implementation modules.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-#include "jam.h"
-#include "execcmd.h"
-
-#include <assert.h>
-#include <stdio.h>
-#include <string.h>
-
-
-/* Internal interrupt counter. */
-static int intr;
-
-
-/* Constructs a list of command-line elements using the format specified by the
- * given shell list.
- *
- * Given argv array should have at least MAXARGC + 1 elements.
- * Slot numbers may be between 0 and 998 (inclusive).
- *
- * Constructed argv list will be zero terminated. Character arrays referenced by
- * the argv structure elements will be either elements from the give shell list,
- * internal static buffers or the given command string and should thus not
- * considered owned by or released via the argv structure and should be
- * considered invalidated by the next argv_from_shell() call.
- *
- * Shell list elements:
- * - Starting with '%' - represent the command string.
- * - Starting with '!' - represent the slot number (increased by one).
- * - Anything else - used as a literal.
- * - If no '%' element is found, the command string is appended as an extra.
- */
-
-void argv_from_shell( char const * * argv, LIST * shell, char const * command,
- int const slot )
-{
- static char jobno[ 4 ];
-
- int i;
- int gotpercent = 0;
- LISTITER iter = list_begin( shell );
- LISTITER end = list_end( shell );
-
- assert( 0 <= slot );
- assert( slot < 999 );
- sprintf( jobno, "%d", slot + 1 );
-
- for ( i = 0; iter != end && i < MAXARGC; ++i, iter = list_next( iter ) )
- {
- switch ( object_str( list_item( iter ) )[ 0 ] )
- {
- case '%': argv[ i ] = command; ++gotpercent; break;
- case '!': argv[ i ] = jobno; break;
- default : argv[ i ] = object_str( list_item( iter ) );
- }
- }
-
- if ( !gotpercent )
- argv[ i++ ] = command;
-
- argv[ i ] = NULL;
-}
-
-
-/* Returns whether the given command string contains lines longer than the given
- * maximum.
- */
-int check_cmd_for_too_long_lines( char const * command, int const max,
- int * const error_length, int * const error_max_length )
-{
- while ( *command )
- {
- size_t const l = strcspn( command, "\n" );
- if ( l > max )
- {
- *error_length = l;
- *error_max_length = max;
- return EXEC_CHECK_LINE_TOO_LONG;
- }
- command += l;
- if ( *command )
- ++command;
- }
- return EXEC_CHECK_OK;
-}
-
-
-/* Checks whether the given shell list is actually a request to execute raw
- * commands without an external shell.
- */
-int is_raw_command_request( LIST * shell )
-{
- return !list_empty( shell ) &&
- !strcmp( object_str( list_front( shell ) ), "%" ) &&
- list_next( list_begin( shell ) ) == list_end( shell );
-}
-
-
-/* Returns whether an interrupt has been detected so far. */
-
-int interrupted( void )
-{
- return intr != 0;
-}
-
-
-/* Internal interrupt handler. */
-
-void onintr( int disp )
-{
- ++intr;
- printf( "...interrupted\n" );
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/execcmd.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/execcmd.h b/ext/kenlm/jam-files/engine/execcmd.h
deleted file mode 100644
index ab145aa..0000000
--- a/ext/kenlm/jam-files/engine/execcmd.h
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Copyright 1993, 1995 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/*
- * execcmd.h - execute a shell script.
- *
- * Defines the interface to be implemented in platform specific implementation
- * modules as well as different shared utility functions prepared in the
- * execcmd.c module.
- */
-
-#ifndef EXECCMD_H
-#define EXECCMD_H
-
-#include "lists.h"
-#include "strings.h"
-#include "timestamp.h"
-
-
-typedef struct timing_info
-{
- double system;
- double user;
- timestamp start;
- timestamp end;
-} timing_info;
-
-typedef void (* ExecCmdCallback)
-(
- void * const closure,
- int const status,
- timing_info const * const,
- char const * const cmd_stdout,
- char const * const cmd_stderr,
- int const cmd_exit_reason
-);
-
-/* Status codes passed to ExecCmdCallback routines. */
-#define EXEC_CMD_OK 0
-#define EXEC_CMD_FAIL 1
-#define EXEC_CMD_INTR 2
-
-int exec_check
-(
- string const * command,
- LIST * * pShell,
- int * error_length,
- int * error_max_length
-);
-
-/* exec_check() return codes. */
-#define EXEC_CHECK_OK 101
-#define EXEC_CHECK_NOOP 102
-#define EXEC_CHECK_LINE_TOO_LONG 103
-#define EXEC_CHECK_TOO_LONG 104
-
-void exec_cmd
-(
- string const * command,
- ExecCmdCallback func,
- void * closure,
- LIST * shell
-);
-
-void exec_wait();
-
-
-/******************************************************************************
- * *
- * Utility functions defined in the execcmd.c module. *
- * *
- ******************************************************************************/
-
-/* Constructs a list of command-line elements using the format specified by the
- * given shell list.
- */
-void argv_from_shell( char const * * argv, LIST * shell, char const * command,
- int const slot );
-
-/* Interrupt routine bumping the internal interrupt counter. Needs to be
- * registered by platform specific exec*.c modules.
- */
-void onintr( int disp );
-
-/* Returns whether an interrupt has been detected so far. */
-int interrupted( void );
-
-/* Checks whether the given shell list is actually a request to execute raw
- * commands without an external shell.
- */
-int is_raw_command_request( LIST * shell );
-
-/* Utility worker for exec_check() checking whether all the given command lines
- * are under the specified length limit.
- */
-int check_cmd_for_too_long_lines( char const * command, int const max,
- int * const error_length, int * const error_max_length );
-
-#endif
[33/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/types/objc.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/types/objc.jam b/ext/kenlm/jam-files/boost-build/tools/types/objc.jam
deleted file mode 100644
index 709cbd0..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/types/objc.jam
+++ /dev/null
@@ -1,26 +0,0 @@
-# Copyright Rene Rivera 2008, 2010.
-# Distributed under the Boost Software License, Version 1.0. (See accompanying
-# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
-import type ;
-import scanner ;
-import types/cpp ;
-
-class objc-scanner : c-scanner
-{
- rule __init__ ( includes * )
- {
- c-scanner.__init__ $(includes) ;
- }
-
- rule pattern ( )
- {
- return "#[ \t]*include|import[ ]*(<(.*)>|\"(.*)\")" ;
- }
-}
-
-scanner.register objc-scanner : include ;
-
-type.register OBJECTIVE_C : m ;
-type.register OBJECTIVE_CPP : mm ;
-type.set-scanner OBJECTIVE_C : objc-scanner ;
-type.set-scanner OBJECTIVE_CPP : objc-scanner ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/types/preprocessed.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/types/preprocessed.jam b/ext/kenlm/jam-files/boost-build/tools/types/preprocessed.jam
deleted file mode 100644
index c9187ba..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/types/preprocessed.jam
+++ /dev/null
@@ -1,9 +0,0 @@
-# Copyright Steven Watanabe 2011
-# Distributed under the Boost Software License Version 1.0. (See
-# accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-import type ;
-
-type.register PREPROCESSED_C : i : C ;
-type.register PREPROCESSED_CPP : ii : CPP ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/types/qt.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/types/qt.jam b/ext/kenlm/jam-files/boost-build/tools/types/qt.jam
deleted file mode 100644
index 4951063..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/types/qt.jam
+++ /dev/null
@@ -1,12 +0,0 @@
-# Copyright Vladimir Prus 2005. Distributed under the Boost
-# Software License, Version 1.0. (See accompanying
-# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
-
-type UI : ui ;
-type QRC : qrc ;
-type MOCCABLE_CPP ;
-type MOCCABLE_H ;
-type MOCCABLE5_CPP ;
-type MOCCABLE5_H ;
-# Result of running moc.
-type MOC : moc : H ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/types/register.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/types/register.jam b/ext/kenlm/jam-files/boost-build/tools/types/register.jam
deleted file mode 100644
index 203992c..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/types/register.jam
+++ /dev/null
@@ -1,39 +0,0 @@
-# Copyright David Abrahams 2004. Distributed under the Boost
-# Software License, Version 1.0. (See accompanying
-# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
-
-# This module's job is to automatically import all the type
-# registration modules in its directory.
-import type os path modules ;
-
-# Register the given type on the specified OSes, or on remaining OSes
-# if os is not specified. This rule is injected into each of the type
-# modules for the sake of convenience.
-local rule type ( type : suffixes * : base-type ? : os * )
-{
- if ! [ type.registered $(type) ]
- {
- if ( ! $(os) ) || [ os.name ] in $(os)
- {
- type.register $(type) : $(suffixes) : $(base-type) ;
- }
- }
-}
-
-.this-module's-file = [ modules.binding $(__name__) ] ;
-.this-module's-dir = [ path.parent $(.this-module's-file) ] ;
-.sibling-jamfiles = [ path.glob $(.this-module's-dir) : *.jam ] ;
-.sibling-modules = [ MATCH ^(.*)\.jam$ : $(.sibling-jamfiles) ] ;
-
-# A loop over all modules in this directory
-for m in $(.sibling-modules)
-{
- m = [ path.basename $(m) ] ;
- m = types/$(m) ;
-
- # Inject the type rule into the new module
- IMPORT $(__name__) : type : $(m) : type ;
- import $(m) ;
-}
-
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/types/rsp.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/types/rsp.jam b/ext/kenlm/jam-files/boost-build/tools/types/rsp.jam
deleted file mode 100644
index bdf8a7c..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/types/rsp.jam
+++ /dev/null
@@ -1,4 +0,0 @@
-# Copyright David Abrahams 2004. Distributed under the Boost
-# Software License, Version 1.0. (See accompanying
-# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
-type RSP : rsp ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/unix.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/unix.jam b/ext/kenlm/jam-files/boost-build/tools/unix.jam
deleted file mode 100644
index 7594985..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/unix.jam
+++ /dev/null
@@ -1,224 +0,0 @@
-# Copyright (c) 2004 Vladimir Prus.
-#
-# Use, modification and distribution is subject to the Boost Software
-# License Version 1.0. (See accompanying file LICENSE_1_0.txt or
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# This file implements linking semantic common to all unixes. On unix, static
-# libraries must be specified in a fixed order on the linker command line. Generators
-# declared there store information about the order and use it property.
-
-import feature ;
-import "class" : new ;
-import generators ;
-import type ;
-import set ;
-import order ;
-import builtin ;
-
-class unix-linking-generator : linking-generator
-{
- import property-set ;
- import type ;
- import unix ;
-
- rule __init__ ( id
- composing ? : # Specify if generator is composing. The generator will be
- # composing if non-empty string is passed, or parameter is
- # not given. To make generator non-composing, pass empty
- # string ("")
- source-types + : target-types + :
- requirements * )
- {
- composing ?= true ;
- generator.__init__ $(id) $(composing) : $(source-types) : $(target-types) :
- $(requirements) ;
- }
-
- rule run ( project name ? : property-set : sources + )
- {
- local result = [ linking-generator.run $(project) $(name) : $(property-set)
- : $(sources) ] ;
-
- unix.set-library-order $(sources) : $(property-set) : $(result[2-]) ;
-
- return $(result) ;
- }
-
- rule generated-targets ( sources + : property-set : project name ? )
- {
- local sources2 ;
- local libraries ;
- for local l in $(sources)
- {
- if [ type.is-derived [ $(l).type ] LIB ]
- {
- libraries += $(l) ;
- }
- else
- {
- sources2 += $(l) ;
- }
- }
-
- sources = $(sources2) [ unix.order-libraries $(libraries) ] ;
-
- return [ linking-generator.generated-targets $(sources) : $(property-set)
- : $(project) $(name) ] ;
- }
-
-}
-
-class unix-archive-generator : archive-generator
-{
- import unix ;
-
- rule __init__ ( id composing ? : source-types + : target-types + :
- requirements * )
- {
- composing ?= true ;
- archive-generator.__init__ $(id) $(composing) : $(source-types) : $(target-types) :
- $(requirements) ;
- }
-
- rule run ( project name ? : property-set : sources + )
- {
- local result = [ archive-generator.run $(project) $(name) : $(property-set)
- : $(sources) ] ;
-
- unix.set-library-order $(sources) : $(property-set) : $(result[2-]) ;
-
- return $(result) ;
-
- }
-}
-
-class unix-searched-lib-generator : searched-lib-generator
-{
- import unix ;
- rule __init__ ( * : * )
- {
- generator.__init__
- $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ;
- }
-
- rule optional-properties ( )
- {
- return $(self.requirements) ;
- }
-
- rule run ( project name ? : property-set : sources * )
- {
- local result = [ searched-lib-generator.run $(project) $(name)
- : $(property-set) : $(sources) ] ;
-
- unix.set-library-order $(sources) : $(property-set) : $(result[2-]) ;
-
- return $(result) ;
- }
-}
-
-class unix-prebuilt-lib-generator : generator
-{
- import unix ;
- rule __init__ ( * : * )
- {
- generator.__init__ $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ;
- }
-
- rule run ( project name ? : property-set : sources * )
- {
- local f = [ $(property-set).get <file> ] ;
- unix.set-library-order-aux $(f) : $(sources) ;
- return $(f) $(sources) ;
- }
-}
-
-generators.register
- [ new unix-prebuilt-lib-generator unix.prebuilt : : LIB
- : <file> <toolset>unix ] ;
-
-generators.override unix.prebuilt : builtin.lib-generator ;
-
-
-# Declare generators
-generators.register [ new unix-linking-generator unix.link : LIB OBJ : EXE
- : <toolset>unix ] ;
-
-generators.register [ new unix-archive-generator unix.archive : OBJ : STATIC_LIB
- : <toolset>unix ] ;
-
-generators.register [ new unix-linking-generator unix.link.dll : LIB OBJ : SHARED_LIB
- : <toolset>unix ] ;
-
-generators.register [ new unix-searched-lib-generator
- unix.searched-lib-generator : : SEARCHED_LIB : <toolset>unix ] ;
-
-
-# The derived toolset must specify their own actions.
-actions link {
-}
-
-actions link.dll {
-}
-
-actions archive {
-}
-
-actions searched-lib-generator {
-}
-
-actions prebuilt {
-}
-
-
-
-
-
-.order = [ new order ] ;
-
-rule set-library-order-aux ( from * : to * )
-{
- for local f in $(from)
- {
- for local t in $(to)
- {
- if $(f) != $(t)
- {
- $(.order).add-pair $(f) $(t) ;
- }
- }
- }
-}
-
-rule set-library-order ( sources * : property-set : result * )
-{
- local used-libraries ;
- local deps = [ $(property-set).dependency ] ;
- for local l in $(sources) $(deps:G=)
- {
- if [ $(l).type ] && [ type.is-derived [ $(l).type ] LIB ]
- {
- used-libraries += $(l) ;
- }
- }
-
- local created-libraries ;
- for local l in $(result)
- {
- if [ $(l).type ] && [ type.is-derived [ $(l).type ] LIB ]
- {
- created-libraries += $(l) ;
- }
- }
-
- created-libraries = [ set.difference $(created-libraries) : $(used-libraries) ] ;
- set-library-order-aux $(created-libraries) : $(used-libraries) ;
-}
-
-rule order-libraries ( libraries * )
-{
- local r = [ $(.order).order $(libraries) ] ;
- return $(r) ;
-}
-
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/vacpp.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/vacpp.jam b/ext/kenlm/jam-files/boost-build/tools/vacpp.jam
deleted file mode 100644
index f4080fc..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/vacpp.jam
+++ /dev/null
@@ -1,150 +0,0 @@
-# Copyright Vladimir Prus 2004.
-# Copyright Toon Knapen 2004.
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt
-# or copy at http://www.boost.org/LICENSE_1_0.txt)
-
-#
-# Boost.Build V2 toolset for the IBM XL C++ compiler
-#
-
-import toolset : flags ;
-import feature ;
-import common ;
-import generators ;
-import os ;
-
-feature.extend toolset : vacpp ;
-toolset.inherit vacpp : unix ;
-generators.override vacpp.prebuilt : builtin.prebuilt ;
-generators.override vacpp.searched-lib-generator : searched-lib-generator ;
-
-# Configure the vacpp toolset
-rule init ( version ? : command * : options * )
-{
- local condition = [
- common.check-init-parameters vacpp : version $(version) ] ;
-
- command = [ common.get-invocation-command vacpp : xlC
- : $(command) : "/usr/vacpp/bin/xlC" ] ;
-
- common.handle-options vacpp : $(condition) : $(command) : $(options) ;
-}
-
-# Declare generators
-generators.register-c-compiler vacpp.compile.c : C : OBJ : <toolset>vacpp ;
-generators.register-c-compiler vacpp.compile.c++ : CPP : OBJ : <toolset>vacpp ;
-
-# Allow C++ style comments in C files
-flags vacpp CFLAGS : -qcpluscmt ;
-
-# Declare flags
-flags vacpp CFLAGS <optimization>off : -qNOOPTimize ;
-flags vacpp CFLAGS <optimization>speed : -O3 -qstrict ;
-flags vacpp CFLAGS <optimization>space : -O2 -qcompact ;
-
-# Discretionary inlining (not recommended)
-flags vacpp CFLAGS <inlining>off : -qnoinline ;
-flags vacpp CFLAGS <inlining>on : -qinline ;
-#flags vacpp CFLAGS <inlining>full : -qinline ;
-flags vacpp CFLAGS <inlining>full : ;
-
-# Exception handling
-flags vacpp C++FLAGS <exception-handling>off : -qnoeh ;
-flags vacpp C++FLAGS <exception-handling>on : -qeh ;
-
-# Run-time Type Identification
-flags vacpp C++FLAGS <rtti>off : -qnortti ;
-flags vacpp C++FLAGS <rtti>on : -qrtti ;
-
-# Enable 64-bit memory addressing model
-flags vacpp CFLAGS <address-model>64 : -q64 ;
-flags vacpp LINKFLAGS <address-model>64 : -q64 ;
-flags vacpp ARFLAGS <target-os>aix/<address-model>64 : -X 64 ;
-
-# Use absolute path when generating debug information
-flags vacpp CFLAGS <debug-symbols>on : -g -qfullpath ;
-flags vacpp LINKFLAGS <debug-symbols>on : -g -qfullpath ;
-flags vacpp LINKFLAGS <debug-symbols>off : -s ;
-
-if [ os.name ] = AIX
-{
- flags vacpp.compile C++FLAGS : -qfuncsect ;
-
- # The -bnoipath strips the prepending (relative) path of libraries from
- # the loader section in the target library or executable. Hence, during
- # load-time LIBPATH (identical to LD_LIBRARY_PATH) or a hard-coded
- # -blibpath (*similar* to -lrpath/-lrpath-link) is searched. Without
- # this option, the prepending (relative) path + library name is
- # hard-coded in the loader section, causing *only* this path to be
- # searched during load-time. Note that the AIX linker does not have an
- # -soname equivalent, this is as close as it gets.
- #
- # The above options are definately for AIX 5.x, and most likely also for
- # AIX 4.x and AIX 6.x. For details about the AIX linker see:
- # http://download.boulder.ibm.com/ibmdl/pub/software/dw/aix/es-aix_ll.pdf
- #
- flags vacpp.link LINKFLAGS <link>shared : -bnoipath ;
-
- # Run-time linking
- flags vacpp.link EXE-LINKFLAGS <link>shared : -brtl ;
-}
-else
-{
- # Linux PPC
- flags vacpp.compile CFLAGS <link>shared : -qpic=large ;
- flags vacpp FINDLIBS : rt ;
-}
-
-# Profiling
-flags vacpp CFLAGS <profiling>on : -pg ;
-flags vacpp LINKFLAGS <profiling>on : -pg ;
-
-flags vacpp.compile OPTIONS <cflags> ;
-flags vacpp.compile.c++ OPTIONS <cxxflags> ;
-flags vacpp DEFINES <define> ;
-flags vacpp UNDEFS <undef> ;
-flags vacpp HDRS <include> ;
-flags vacpp STDHDRS <sysinclude> ;
-flags vacpp.link OPTIONS <linkflags> ;
-flags vacpp ARFLAGS <arflags> ;
-
-flags vacpp LIBPATH <library-path> ;
-flags vacpp NEEDLIBS <library-file> ;
-flags vacpp FINDLIBS <find-shared-library> ;
-flags vacpp FINDLIBS <find-static-library> ;
-
-# Select the compiler name according to the threading model.
-flags vacpp VA_C_COMPILER <threading>single : xlc ;
-flags vacpp VA_C_COMPILER <threading>multi : xlc_r ;
-flags vacpp VA_CXX_COMPILER <threading>single : xlC ;
-flags vacpp VA_CXX_COMPILER <threading>multi : xlC_r ;
-
-SPACE = " " ;
-
-flags vacpp.link.dll HAVE_SONAME <target-os>linux : "" ;
-
-actions vacpp.link bind NEEDLIBS
-{
- $(VA_CXX_COMPILER) $(EXE-LINKFLAGS) $(LINKFLAGS) -o "$(<[1])" -L$(LIBPATH) -L$(STDLIBPATH) "$(>)" "$(NEEDLIBS)" "$(NEEDLIBS)" -l$(FINDLIBS) $(OPTIONS) $(USER_OPTIONS)
-}
-
-actions vacpp.link.dll bind NEEDLIBS
-{
- xlC_r -G $(LINKFLAGS) -o "$(<[1])" $(HAVE_SONAME)-Wl,-soname$(SPACE)-Wl,$(<[-1]:D=) -L$(LIBPATH) -L$(STDLIBPATH) "$(>)" "$(NEEDLIBS)" "$(NEEDLIBS)" -l$(FINDLIBS) $(OPTIONS) $(USER_OPTIONS)
-}
-
-actions vacpp.compile.c
-{
- $(VA_C_COMPILER) -c $(OPTIONS) $(USER_OPTIONS) -I$(BOOST_ROOT) -U$(UNDEFS) -D$(DEFINES) $(CFLAGS) -I"$(HDRS)" -I"$(STDHDRS)" -o "$(<)" "$(>)"
-}
-
-actions vacpp.compile.c++
-{
- $(VA_CXX_COMPILER) -c $(OPTIONS) $(USER_OPTIONS) -I$(BOOST_ROOT) -U$(UNDEFS) -D$(DEFINES) $(CFLAGS) $(C++FLAGS) -I"$(HDRS)" -I"$(STDHDRS)" -o "$(<)" "$(>)"
-}
-
-actions updated together piecemeal vacpp.archive
-{
- ar $(ARFLAGS) ru "$(<)" "$(>)"
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/whale.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/whale.jam b/ext/kenlm/jam-files/boost-build/tools/whale.jam
deleted file mode 100644
index 9335ff0..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/whale.jam
+++ /dev/null
@@ -1,116 +0,0 @@
-# Copyright (C) Vladimir Prus 2002-2005.
-
-# Use, modification and distribution is subject to the Boost Software
-# License Version 1.0. (See accompanying file LICENSE_1_0.txt or
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# This module implements support for Whale/Dolphin/WD parser/lexer tools.
-# See http://www.cs.queensu.ca/home/okhotin/whale/ for details.
-#
-# There are three interesting target types:
-# - WHL (the parser sources), that are converted to CPP and H
-# - DLP (the lexer sources), that are converted to CPP and H
-# - WD (combined parser/lexer sources), that are converted to WHL + DLP
-
-import type ;
-import generators ;
-import path ;
-import "class" : new ;
-import errors ;
-
-rule init ( path # path the Whale/Dolphin/WD binaries
- )
-{
- if $(.configured) && $(.path) != $(path)
- {
- errors.user-error "Attempt to reconfigure Whale support" :
- "Previously configured with path \"$(.path:E=<empty>)\"" :
- "Now configuring with path \"$(path:E=<empty>)\"" ;
-
- }
- .configured = true ;
- .path = $(path) ;
-
- .whale = [ path.join $(path) whale ] ;
- .dolphin = [ path.join $(path) dolphin ] ;
- .wd = [ path.join $(path) wd ] ;
-}
-
-
-# Declare the types.
-type.register WHL : whl ;
-type.register DLP : dlp ;
-type.register WHL_LR0 : lr0 ;
-type.register WD : wd ;
-
-# Declare standard generators.
-generators.register-standard whale.whale : WHL : CPP H H(%_symbols) ;
-generators.register-standard whale.dolphin : DLP : CPP H ;
-generators.register-standard whale.wd : WD : WHL(%_parser) DLP(%_lexer) ;
-
-# The conversions defines above a ambiguious when we generated CPP from WD.
-# We can either go via WHL type, or via DLP type.
-# The following custom generator handles this by running both conversions.
-
-class wd-to-cpp : generator
-{
- rule __init__ ( * : * : * )
- {
- generator.__init__ $(1) : $(2) : $(3) ;
- }
-
- rule run ( project name ? : property-set : source * )
- {
- if ! $(source[2])
- {
- local new-sources ;
- if ! [ $(source).type ] in WHL DLP
- {
- local r1 = [ generators.construct $(project) $(name)
- : WHL : $(property-set) : $(source) ] ;
- local r2 = [ generators.construct $(project) $(name)
- : DLP : $(property-set) : $(source) ] ;
-
- new-sources = [ sequence.unique $(r1[2-]) $(r2[2-]) ] ;
- }
- else
- {
- new-sources = $(source) ;
- }
-
- local result ;
- for local i in $(new-sources)
- {
- local t = [ generators.construct $(project) $(name) : CPP
- : $(property-set) : $(i) ] ;
- result += $(t[2-]) ;
- }
- return $(result) ;
- }
- }
-
-}
-
-
-generators.override whale.wd-to-cpp : whale.whale ;
-generators.override whale.wd-to-cpp : whale.dolphin ;
-
-
-generators.register [ new wd-to-cpp whale.wd-to-cpp : : CPP ] ;
-
-
-actions whale
-{
- $(.whale) -d $(<[1]:D) $(>)
-}
-
-actions dolphin
-{
- $(.dolphin) -d $(<[1]:D) $(>)
-}
-
-actions wd
-{
- $(.wd) -d $(<[1]:D) -g $(>)
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/xlf.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/xlf.jam b/ext/kenlm/jam-files/boost-build/tools/xlf.jam
deleted file mode 100644
index e7fcc60..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/xlf.jam
+++ /dev/null
@@ -1,39 +0,0 @@
-# Copyright (C) 2004 Toon Knapen
-#
-# Use, modification and distribution is subject to the Boost Software
-# License Version 1.0. (See accompanying file LICENSE_1_0.txt or
-# http://www.boost.org/LICENSE_1_0.txt)
-
-#
-# toolset configuration for the IBM Fortran compiler (xlf)
-#
-
-import toolset : flags ;
-import feature ;
-import fortran ;
-
-rule init ( version ? : command * : options * )
-{
-}
-
-# Declare flags and action for compilation
-flags xlf OPTIONS <optimization>off : -O0 ;
-flags xlf OPTIONS <optimization>speed : -O3 ;
-flags xlf OPTIONS <optimization>space : -Os ;
-
-flags xlf OPTIONS <debug-symbols>on : -g ;
-flags xlf OPTIONS <profiling>on : -pg ;
-
-flags xlf DEFINES <define> ;
-flags xlf INCLUDES <include> ;
-
-rule compile-fortran
-{
-}
-
-actions compile-fortran
-{
- xlf $(OPTIONS) -I$(INCLUDES) -c -o "$(<)" "$(>)"
-}
-
-generators.register-fortran-compiler xlf.compile-fortran : FORTRAN : OBJ ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/xsltproc-config.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/xsltproc-config.jam b/ext/kenlm/jam-files/boost-build/tools/xsltproc-config.jam
deleted file mode 100644
index d1be25f..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/xsltproc-config.jam
+++ /dev/null
@@ -1,36 +0,0 @@
-#~ Copyright 2005 Rene Rivera.
-#~ Distributed under the Boost Software License, Version 1.0.
-#~ (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-# Automatic configuration for the xsltproc toolset. To use, just import this
-# module.
-
-import os ;
-import toolset : using ;
-
-
-local rule locate-executable ( name )
-{
- local path = [ modules.peek : PATH ] ;
- local exe ;
- if [ os.name ] = NT
- {
- exe = [ GLOB $(path) "C:\\Boost\\bin" : $(name)\.exe ] ;
- }
- else
- {
- exe = [ GLOB $(path) : $(name) ] ;
- }
- return $(exe[1]) ;
-}
-
-
-local xsltproc-exe = [ locate-executable xsltproc ] ;
-if $(xsltproc-exe)
-{
- if --debug-configuration in [ modules.peek : ARGV ]
- {
- ECHO notice: using xsltproc ":" $(xsltproc-exe) ;
- }
- using xsltproc : $(xsltproc-exe) ;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/xsltproc.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/xsltproc.jam b/ext/kenlm/jam-files/boost-build/tools/xsltproc.jam
deleted file mode 100644
index d847646..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/xsltproc.jam
+++ /dev/null
@@ -1,205 +0,0 @@
-# Copyright (C) 2003 Doug Gregor. Permission to copy, use, modify, sell and
-# distribute this software is granted provided this copyright notice appears in
-# all copies. This software is provided "as is" without express or implied
-# warranty, and with no claim as to its suitability for any purpose.
-
-# This module defines rules to apply an XSLT stylesheet to an XML file using the
-# xsltproc driver, part of libxslt.
-
-import common ;
-import feature ;
-import modules ;
-import os ;
-import path ;
-import regex ;
-import sequence ;
-
-
-feature.feature xsl:param : : free ;
-feature.feature xsl:path : : free ;
-feature.feature catalog : : free ;
-
-
-# Initialize xsltproc support. The parameters are:
-# xsltproc: The xsltproc executable
-#
-rule init ( xsltproc ? )
-{
- if $(xsltproc)
- {
- modify-config ;
- .xsltproc = $(xsltproc) ;
- check-xsltproc ;
- }
-}
-
-
-rule freeze-config ( )
-{
- if ! $(.config-frozen)
- {
- .config-frozen = true ;
- .xsltproc ?= [ modules.peek : XSLTPROC ] ;
- .xsltproc ?= xsltproc ;
- check-xsltproc ;
- .is-cygwin = [ .is-cygwin $(.xsltproc) ] ;
- }
-}
-
-
-rule modify-config ( )
-{
- if $(.config-frozen)
- {
- import errors ;
- errors.user-error
- "xsltproc: Cannot change xsltproc command after it has been used." ;
- }
-}
-
-
-rule check-xsltproc ( )
-{
- if $(.xsltproc)
- {
- local status = [ SHELL "\"$(.xsltproc)\" -V" : no-output : exit-status ]
- ;
- if $(status[2]) != 0
- {
- import errors ;
- errors.user-error "xsltproc: Could not run \"$(.xsltproc)\" -V." ;
- }
- }
-}
-
-
-# Returns a non-empty string if a cygwin xsltproc binary was specified.
-#
-rule is-cygwin ( )
-{
- freeze-config ;
- return $(.is-cygwin) ;
-}
-
-
-rule .is-cygwin ( xsltproc )
-{
- if [ os.on-windows ]
- {
- local file = [ path.make [ modules.binding $(__name__) ] ] ;
- local dir = [ path.native [ path.join [ path.parent $(file) ] xsltproc ]
- ] ;
- if [ os.name ] = CYGWIN
- {
- dir = $(dir:W) ;
- }
- local command =
- "\"$(xsltproc)\" \"$(dir)\\test.xsl\" \"$(dir)\\test.xml\" 2>&1" ;
- local status = [ SHELL $(command) : no-output : exit-status ] ;
- if $(status[2]) != "0"
- {
- return true ;
- }
- }
-}
-
-
-rule compute-xslt-flags ( target : properties * )
-{
- # Raw flags.
- local flags = [ feature.get-values <flags> : $(properties) ] ;
-
- # Translate <xsl:param> into command line flags.
- for local param in [ feature.get-values <xsl:param> : $(properties) ]
- {
- local namevalue = [ regex.split $(param) "=" ] ;
- flags += --stringparam $(namevalue[1]) \"$(namevalue[2])\" ;
- }
-
- # Translate <xsl:path>.
- for local path in [ feature.get-values <xsl:path> : $(properties) ]
- {
- flags += --path \"$(path:G=)\" ;
- }
-
- # Take care of implicit dependencies.
- local other-deps ;
- for local dep in [ feature.get-values <implicit-dependency> : $(properties)
- ]
- {
- other-deps += [ $(dep:G=).creating-subvariant ] ;
- }
-
- local implicit-target-directories ;
- for local dep in [ sequence.unique $(other-deps) ]
- {
- implicit-target-directories += [ $(dep).all-target-directories ] ;
- }
-
- for local dir in $(implicit-target-directories)
- {
- flags += --path \"$(dir:T)\" ;
- }
-
- return $(flags) ;
-}
-
-
-local rule .xsltproc ( target : source stylesheet : properties * : dirname ? :
- action )
-{
- freeze-config ;
- STYLESHEET on $(target) = $(stylesheet) ;
- FLAGS on $(target) += [ compute-xslt-flags $(target) : $(properties) ] ;
- NAME on $(target) = $(.xsltproc) ;
-
- for local catalog in [ feature.get-values <catalog> : $(properties) ]
- {
- CATALOG = [ common.variable-setting-command XML_CATALOG_FILES :
- $(catalog:T) ] ;
- }
-
- if [ os.on-windows ] && ! [ is-cygwin ]
- {
- action = $(action).windows ;
- }
-
- $(action) $(target) : $(source) ;
-}
-
-
-rule xslt ( target : source stylesheet : properties * )
-{
- return [ .xsltproc $(target) : $(source) $(stylesheet) : $(properties) : :
- xslt-xsltproc ] ;
-}
-
-
-rule xslt-dir ( target : source stylesheet : properties * : dirname )
-{
- return [ .xsltproc $(target) : $(source) $(stylesheet) : $(properties) :
- $(dirname) : xslt-xsltproc-dir ] ;
-}
-
-actions xslt-xsltproc.windows
-{
- $(CATALOG) "$(NAME:E=xsltproc)" $(FLAGS) --xinclude -o "$(<)" "$(STYLESHEET:W)" "$(>:W)"
-}
-
-
-actions xslt-xsltproc bind STYLESHEET
-{
- $(CATALOG) "$(NAME:E=xsltproc)" $(FLAGS) --xinclude -o "$(<)" "$(STYLESHEET:T)" "$(>:T)"
-}
-
-
-actions xslt-xsltproc-dir.windows bind STYLESHEET
-{
- $(CATALOG) "$(NAME:E=xsltproc)" $(FLAGS) --xinclude -o "$(<:D)/" "$(STYLESHEET:W)" "$(>:W)"
-}
-
-
-actions xslt-xsltproc-dir bind STYLESHEET
-{
- $(CATALOG) "$(NAME:E=xsltproc)" $(FLAGS) --xinclude -o "$(<:D)/" "$(STYLESHEET:T)" "$(>:T)"
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/xsltproc/included.xsl
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/xsltproc/included.xsl b/ext/kenlm/jam-files/boost-build/tools/xsltproc/included.xsl
deleted file mode 100644
index ef86394..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/xsltproc/included.xsl
+++ /dev/null
@@ -1,11 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!--
- Copyright (c) 2010 Steven Watanabe
-
- Distributed under the Boost Software License, Version 1.0.
- (See accompanying file LICENSE_1_0.txt or copy at
- http://www.boost.org/LICENSE_1_0.txt)
- -->
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
- version="1.0">
-</xsl:stylesheet>
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/xsltproc/test.xml
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/xsltproc/test.xml b/ext/kenlm/jam-files/boost-build/tools/xsltproc/test.xml
deleted file mode 100644
index 57c8ba1..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/xsltproc/test.xml
+++ /dev/null
@@ -1,2 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<root/>
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/xsltproc/test.xsl
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/xsltproc/test.xsl b/ext/kenlm/jam-files/boost-build/tools/xsltproc/test.xsl
deleted file mode 100644
index a142c91..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/xsltproc/test.xsl
+++ /dev/null
@@ -1,12 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!--
- Copyright (c) 2010 Steven Watanabe
-
- Distributed under the Boost Software License, Version 1.0.
- (See accompanying file LICENSE_1_0.txt or copy at
- http://www.boost.org/LICENSE_1_0.txt)
- -->
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
- version="1.0">
- <xsl:include href="included.xsl"/>
-</xsl:stylesheet>
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/zlib.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/zlib.jam b/ext/kenlm/jam-files/boost-build/tools/zlib.jam
deleted file mode 100644
index 8095eee..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/zlib.jam
+++ /dev/null
@@ -1,227 +0,0 @@
-# Copyright (c) 2010 Vladimir Prus.
-# Copyright (c) 2013 Steven Watanabe
-#
-# Use, modification and distribution is subject to the Boost Software
-# License Version 1.0. (See accompanying file LICENSE_1_0.txt or
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# Supports the zlib library
-#
-# After 'using zlib', the following targets are available:
-#
-# /zlib//zlib -- The zlib library
-
-import project ;
-import ac ;
-import errors ;
-import "class" : new ;
-import targets ;
-import path ;
-import modules ;
-import errors ;
-import indirect ;
-import property ;
-import property-set ;
-
-header = zlib.h ;
-names = z zlib zll zdll ;
-
-sources = adler32.c compress.c
- crc32.c deflate.c gzclose.c gzio.c gzlib.c gzread.c gzwrite.c
- infback.c inffast.c inflate.c inftrees.c trees.c uncompr.c zutil.c ;
-
-library-id = 0 ;
-
-if --debug-configuration in [ modules.peek : ARGV ]
-{
- .debug = true ;
-}
-
-# Initializes the zlib library.
-#
-# zlib can be configured either to use pre-existing binaries
-# or to build the library from source.
-#
-# Options for configuring a prebuilt zlib::
-#
-# <search>
-# The directory containing the zlib binaries.
-# <name>
-# Overrides the default library name.
-# <include>
-# The directory containing the zlib headers.
-#
-# If none of these options is specified, then the environmental
-# variables ZLIB_LIBRARY_PATH, ZLIB_NAME, and ZLIB_INCLUDE will
-# be used instead.
-#
-# Options for building zlib from source::
-#
-# <source>
-# The zlib source directory. Defaults to the environmental variable
-# ZLIB_SOURCE.
-# <tag>
-# A rule which computes the actual name of the compiled
-# libraries based on the build properties. Ignored
-# when using precompiled binaries.
-# <build-name>
-# The base name to use for the compiled library. Ignored
-# when using precompiled binaries.
-#
-# Examples::
-#
-# # Find zlib in the default system location
-# using zlib ;
-# # Build zlib from source
-# using zlib : 1.2.7 : <source>/home/steven/zlib-1.2.7 ;
-# # Find zlib in /usr/local
-# using zlib : 1.2.7
-# : <include>/usr/local/include <search>/usr/local/lib ;
-# # Build zlib from source for msvc and find
-# # prebuilt binaries for gcc.
-# using zlib : 1.2.7 : <source>C:/Devel/src/zlib-1.2.7 : <toolset>msvc ;
-# using zlib : 1.2.7 : : <toolset>gcc ;
-#
-rule init (
- version ?
- # The zlib version (currently ignored)
-
- : options *
- # A list of the options to use
-
- : requirements *
- # The requirements for the zlib target
-
- : is-default ?
- # Default configurations are only used when zlib
- # has not yet been configured.
- )
-{
- local caller = [ project.current ] ;
-
- if ! $(.initialized)
- {
- .initialized = true ;
-
- project.initialize $(__name__) ;
- .project = [ project.current ] ;
- project zlib ;
- }
-
- local library-path = [ property.select <search> : $(options) ] ;
- library-path = $(library-path:G=) ;
- local include-path = [ property.select <include> : $(options) ] ;
- include-path = $(include-path:G=) ;
- local source-path = [ property.select <source> : $(options) ] ;
- source-path = $(source-path:G=) ;
- local library-name = [ property.select <name> : $(options) ] ;
- library-name = $(library-name:G=) ;
- local tag = [ property.select <tag> : $(options) ] ;
- tag = $(tag:G=) ;
- local build-name = [ property.select <build-name> : $(options) ] ;
- build-name = $(build-name:G=) ;
-
- condition = [ property-set.create $(requirements) ] ;
- condition = [ property-set.create [ $(condition).base ] ] ;
-
- local no-build-from-source ;
- # Ignore environmental ZLIB_SOURCE if this initialization
- # requested to search for a specific pre-built library.
- if $(library-path) || $(include-path) || $(library-name)
- {
- if $(source-path) || $(tag) || $(build-name)
- {
- errors.user-error "incompatible options for zlib:"
- [ property.select <search> <include> <name> : $(options) ] "and"
- [ property.select <source> <tag> <build-name> : $(options) ] ;
- }
- else
- {
- no-build-from-source = true ;
- }
- }
-
- source-path ?= [ modules.peek : ZLIB_SOURCE ] ;
-
- if $(.configured.$(condition))
- {
- if $(is-default)
- {
- if $(.debug)
- {
- ECHO "notice: [zlib] zlib is already configured" ;
- }
- }
- else
- {
- errors.user-error "zlib is already configured" ;
- }
- return ;
- }
- else if $(source-path) && ! $(no-build-from-source)
- {
- build-name ?= z ;
- library-id = [ CALC $(library-id) + 1 ] ;
- tag = [ MATCH ^@?(.*)$ : $(tag) ] ;
- if $(tag) && ! [ MATCH ^([^%]*)%([^%]+)$ : $(tag) ]
- {
- tag = [ indirect.make $(tag) : [ $(caller).project-module ] ] ;
- }
- sources = [ path.glob $(source-path) : $(sources) ] ;
- if $(.debug)
- {
- ECHO "notice: [zlib] Building zlib from source as $(build-name)" ;
- if $(condition)
- {
- ECHO "notice: [zlib] Condition" [ $(condition).raw ] ;
- }
- if $(sources)
- {
- ECHO "notice: [zlib] found zlib source in $(source-path)" ;
- }
- else
- {
- ECHO "warning: [zlib] could not find zlib source in $(source-path)" ;
- }
- }
- local target ;
- if $(sources) {
- target = [ targets.create-typed-target LIB : $(.project)
- : $(build-name).$(library-id)
- : $(sources)
- : $(requirements)
- <tag>@$(tag)
- <include>$(source-path)
- <toolset>msvc:<define>_CRT_SECURE_NO_DEPRECATE
- <toolset>msvc:<define>_SCL_SECURE_NO_DEPRECATE
- <link>shared:<define>ZLIB_DLL
- :
- : <include>$(source-path) ] ;
- }
-
- local mt = [ new ac-library zlib : $(.project) : $(condition) ] ;
- $(mt).set-header $(header) ;
- $(mt).set-default-names $(names) ;
- if $(target)
- {
- $(mt).set-target $(target) ;
- }
- targets.main-target-alternative $(mt) ;
- } else {
- if $(.debug)
- {
- ECHO "notice: [zlib] Using pre-installed library" ;
- if $(condition)
- {
- ECHO "notice: [zlib] Condition" [ $(condition).raw ] ;
- }
- }
-
- local mt = [ new ac-library zlib : $(.project) : $(condition) :
- $(include-path) : $(library-path) : $(library-name) : $(root) ] ;
- $(mt).set-header $(header) ;
- $(mt).set-default-names $(names) ;
- targets.main-target-alternative $(mt) ;
- }
- .configured.$(condition) = true ;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/user-config.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/user-config.jam b/ext/kenlm/jam-files/boost-build/user-config.jam
deleted file mode 100644
index fbbf13f..0000000
--- a/ext/kenlm/jam-files/boost-build/user-config.jam
+++ /dev/null
@@ -1,92 +0,0 @@
-# Copyright 2003, 2005 Douglas Gregor
-# Copyright 2004 John Maddock
-# Copyright 2002, 2003, 2004, 2007 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-# This file is used to configure your Boost.Build installation. You can modify
-# this file in place, or you can place it in a permanent location so that it
-# does not get overwritten should you get a new version of Boost.Build. See:
-#
-# http://www.boost.org/boost-build2/doc/html/bbv2/overview/configuration.html
-#
-# for documentation about possible permanent locations.
-
-# This file specifies which toolsets (C++ compilers), libraries, and other
-# tools are available. Often, you should be able to just uncomment existing
-# example lines and adjust them to taste. The complete list of supported tools,
-# and configuration instructions can be found at:
-#
-# http://boost.org/boost-build2/doc/html/bbv2/reference/tools.html
-#
-
-# This file uses Jam language syntax to describe available tools. Mostly,
-# there are 'using' lines, that contain the name of the used tools, and
-# parameters to pass to those tools -- where paremeters are separated by
-# semicolons. Important syntax notes:
-#
-# - Both ':' and ';' must be separated from other tokens by whitespace
-# - The '\' symbol is a quote character, so when specifying Windows paths you
-# should use '/' or '\\' instead.
-#
-# More details about the syntax can be found at:
-#
-# http://boost.org/boost-build2/doc/html/bbv2/advanced.html#bbv2.advanced.jam_language
-#
-
-# ------------------
-# GCC configuration.
-# ------------------
-
-# Configure gcc (default version).
-# using gcc ;
-
-# Configure specific gcc version, giving alternative name to use.
-# using gcc : 3.2 : g++-3.2 ;
-
-
-# -------------------
-# MSVC configuration.
-# -------------------
-
-# Configure msvc (default version, searched for in standard locations and PATH).
-# using msvc ;
-
-# Configure specific msvc version (searched for in standard locations and PATH).
-# using msvc : 8.0 ;
-
-
-# ----------------------
-# Borland configuration.
-# ----------------------
-# using borland ;
-
-
-# ----------------------
-# STLPort configuration.
-# ----------------------
-
-# Configure specifying location of STLPort headers. Libraries must be either
-# not needed or available to the compiler by default.
-# using stlport : : /usr/include/stlport ;
-
-# Configure specifying location of both headers and libraries explicitly.
-# using stlport : : /usr/include/stlport /usr/lib ;
-
-
-# -----------------
-# QT configuration.
-# -----------------
-
-# Configure assuming QTDIR gives the installation prefix.
-# using qt ;
-
-# Configure with an explicit installation prefix.
-# using qt : /usr/opt/qt ;
-
-# ---------------------
-# Python configuration.
-# ---------------------
-
-# Configure specific Python version.
-# using python : 3.1 : /usr/bin/python3 : /usr/include/python3.1 : /usr/lib ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/util/assert.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/util/assert.jam b/ext/kenlm/jam-files/boost-build/util/assert.jam
deleted file mode 100644
index 65e880f..0000000
--- a/ext/kenlm/jam-files/boost-build/util/assert.jam
+++ /dev/null
@@ -1,346 +0,0 @@
-# Copyright 2001, 2002, 2003 Dave Abrahams
-# Copyright 2006 Rene Rivera
-# Copyright 2002, 2003 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-import errors ;
-import modules ;
-
-
-################################################################################
-#
-# Private implementation details.
-#
-################################################################################
-
-# Rule added as a replacement for the regular Jam = operator but which does not
-# ignore trailing empty string elements.
-#
-local rule exact-equal-test ( lhs * : rhs * )
-{
- local lhs_extended = $(lhs) xxx ;
- local rhs_extended = $(rhs) xxx ;
- if $(lhs_extended) = $(rhs_extended)
- {
- return true ;
- }
-}
-
-
-# Two lists are considered set-equal if they contain the same elements, ignoring
-# duplicates and ordering.
-#
-local rule set-equal-test ( set1 * : set2 * )
-{
- if ( $(set1) in $(set2) ) && ( $(set2) in $(set1) )
- {
- return true ;
- }
-}
-
-
-################################################################################
-#
-# Public interface.
-#
-################################################################################
-
-# Assert the equality of A and B, ignoring trailing empty string elements.
-#
-rule equal ( a * : b * )
-{
- if $(a) != $(b)
- {
- errors.error-skip-frames 3 assertion failure: \"$(a)\" "==" \"$(b)\"
- (ignoring trailing empty strings) ;
- }
-}
-
-
-# Assert that the result of calling RULE-NAME on the given arguments has a false
-# logical value (is either an empty list or all empty strings).
-#
-rule false ( rule-name args * : * )
-{
- local result ;
- module [ CALLER_MODULE ]
- {
- modules.poke assert : result : [ $(1) : $(2) : $(3) : $(4) : $(5) : $(6)
- : $(7) : $(8) : $(9) : $(10) : $(11) : $(12) : $(13) : $(14) : $(15)
- : $(16) : $(17) : $(18) : $(19) ] ;
- }
-
- if $(result)
- {
- errors.error-skip-frames 3 assertion failure: Expected false result from
- "[" $(rule-name) [ errors.lol->list $(args) : $(2) : $(3) : $(4) :
- $(5) : $(6) : $(7) : $(8) : $(9) : $(10) : $(11) : $(12) : $(13) :
- $(14) : $(15) : $(16) : $(17) : $(18) : $(19) ] "]" : Got: "["
- \"$(result)\" "]" ;
- }
-}
-
-
-# Assert that ELEMENT is present in LIST.
-#
-rule "in" ( element : list * )
-{
- if ! $(element) in $(list)
- {
- errors.error-skip-frames 3 assertion failure: Expected \"$(element)\" in
- "[" \"$(list)\" "]" ;
- }
-}
-
-
-# Assert the inequality of A and B, ignoring trailing empty string elements.
-#
-rule not-equal ( a * : b * )
-{
- if $(a) = $(b)
- {
- errors.error-skip-frames 3 assertion failure: \"$(a)\" "!=" \"$(b)\"
- (ignoring trailing empty strings) ;
- }
-}
-
-
-# Assert that ELEMENT is not present in LIST.
-#
-rule not-in ( element : list * )
-{
- if $(element) in $(list)
- {
- errors.error-skip-frames 3 assertion failure: Did not expect
- \"$(element)\" in "[" \"$(list)\" "]" ;
- }
-}
-
-
-# Assert the inequality of A and B as sets.
-#
-rule not-set-equal ( a * : b * )
-{
- if [ set-equal-test $(a) : $(b) ]
- {
- errors.error-skip-frames 3 assertion failure: Expected "[" \"$(a)\" "]"
- and "[" \"$(b)\" "]" to not be equal as sets ;
- }
-}
-
-
-# Assert that A and B are not exactly equal, not ignoring trailing empty string
-# elements.
-#
-rule not-exact-equal ( a * : b * )
-{
- if [ exact-equal-test $(a) : $(b) ]
- {
- errors.error-skip-frames 3 assertion failure: \"$(a)\" "!=" \"$(b)\" ;
- }
-}
-
-
-# Assert that EXPECTED is the result of calling RULE-NAME with the given
-# arguments.
-#
-rule result ( expected * : rule-name args * : * )
-{
- local result ;
- module [ CALLER_MODULE ]
- {
- modules.poke assert : result : [ $(2) : $(3) : $(4) : $(5) : $(6) : $(7)
- : $(8) : $(9) : $(10) : $(11) : $(12) : $(13) : $(14) : $(15) :
- $(16) : $(17) : $(18) : $(19) ] ;
- }
-
- if ! [ exact-equal-test $(result) : $(expected) ]
- {
- errors.error-skip-frames 3 assertion failure: "[" $(rule-name) [
- errors.lol->list $(args) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) :
- $(9) : $(10) : $(11) : $(12) : $(13) : $(14) : $(15) : $(16) : $(17)
- : $(18) : $(19) ] "]" : Expected: "[" \"$(expected)\" "]" : Got: "["
- \"$(result)\" "]" ;
- }
-}
-
-
-# Assert that EXPECTED is set-equal (i.e. duplicates and ordering are ignored)
-# to the result of calling RULE-NAME with the given arguments. Note that rules
-# called this way may accept at most 18 parameters.
-#
-rule result-set-equal ( expected * : rule-name args * : * )
-{
- local result ;
- module [ CALLER_MODULE ]
- {
- modules.poke assert : result : [ $(2) : $(3) : $(4) : $(5) : $(6) : $(7)
- : $(8) : $(9) : $(10) : $(11) : $(12) : $(13) : $(14) : $(15) :
- $(16) : $(17) : $(18) : $(19) ] ;
- }
-
- if ! [ set-equal-test $(result) : $(expected) ]
- {
- errors.error-skip-frames 3 assertion failure: "[" $(rule-name) [
- errors.lol->list $(args) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) :
- $(9) : $(10) : $(11) : $(12) : $(13) : $(14) : $(15) : $(16) : $(17)
- : $(18) : $(19) ] "]" : Expected: "[" \"$(expected)\" "]" : Got: "["
- \"$(result)\" "]" ;
- }
-}
-
-
-# Assert the equality of A and B as sets.
-#
-rule set-equal ( a * : b * )
-{
- if ! [ set-equal-test $(a) : $(b) ]
- {
- errors.error-skip-frames 3 assertion failure: Expected "[" \"$(a)\" "]"
- and "[" \"$(b)\" "]" to be equal as sets ;
- }
-}
-
-
-# Assert that the result of calling RULE-NAME on the given arguments has a true
-# logical value (is neither an empty list nor all empty strings).
-#
-rule true ( rule-name args * : * )
-{
- local result ;
- module [ CALLER_MODULE ]
- {
- modules.poke assert : result : [ $(1) : $(2) : $(3) : $(4) : $(5) : $(6)
- : $(7) : $(8) : $(9) : $(10) : $(11) : $(12) : $(13) : $(14) : $(15)
- : $(16) : $(17) : $(18) : $(19) ] ;
- }
-
- if ! $(result)
- {
- errors.error-skip-frames 3 assertion failure: Expected true result from
- "[" $(rule-name) [ errors.lol->list $(args) : $(2) : $(3) : $(4) :
- $(5) : $(6) : $(7) : $(8) : $(9) : $(10) : $(11) : $(12) : $(13) :
- $(14) : $(15) : $(16) : $(17) : $(18) : $(19) ] "]" ;
- }
-}
-
-
-# Assert the exact equality of A and B, not ignoring trailing empty string
-# elements.
-#
-rule exact-equal ( a * : b * )
-{
- if ! [ exact-equal-test $(a) : $(b) ]
- {
- errors.error-skip-frames 3 assertion failure: \"$(a)\" "==" \"$(b)\" ;
- }
-}
-
-
-# Assert that the given variable is not an empty list.
-#
-rule variable-not-empty ( name )
-{
- local value = [ modules.peek [ CALLER_MODULE ] : $(name) ] ;
- if ! $(value)-is-not-empty
- {
- errors.error-skip-frames 3 assertion failure: Expected variable
- \"$(name)\" not to be an empty list ;
- }
-}
-
-
-rule __test__ ( )
-{
- # Helper rule used to avoid test duplication related to different list
- # equality test rules.
- #
- local rule run-equality-test ( equality-assert : ignore-trailing-empty-strings ? )
- {
- local not-equality-assert = not-$(equality-assert) ;
-
- # When the given equality test is expected to ignore trailing empty
- # strings some of the test results should be inverted.
- local not-equality-assert-i = not-$(equality-assert) ;
- if $(ignore-trailing-empty-strings)
- {
- not-equality-assert-i = $(equality-assert) ;
- }
-
- $(equality-assert) : ;
- $(equality-assert) "" "" : "" "" ;
- $(not-equality-assert-i) : "" "" ;
- $(equality-assert) x : x ;
- $(not-equality-assert) : x ;
- $(not-equality-assert) "" : x ;
- $(not-equality-assert) "" "" : x ;
- $(not-equality-assert-i) x : x "" ;
- $(equality-assert) x "" : x "" ;
- $(not-equality-assert) x : "" x ;
- $(equality-assert) "" x : "" x ;
-
- $(equality-assert) 1 2 3 : 1 2 3 ;
- $(not-equality-assert) 1 2 3 : 3 2 1 ;
- $(not-equality-assert) 1 2 3 : 1 5 3 ;
- $(not-equality-assert) 1 2 3 : 1 "" 3 ;
- $(not-equality-assert) 1 2 3 : 1 1 2 3 ;
- $(not-equality-assert) 1 2 3 : 1 2 2 3 ;
- $(not-equality-assert) 1 2 3 : 5 6 7 ;
-
- # Extra variables used here just to make sure Boost Jam or Boost Build
- # do not handle lists with empty strings differently depending on
- # whether they are literals or stored in variables.
-
- local empty = ;
- local empty-strings = "" "" ;
- local x-empty-strings = x "" "" ;
- local empty-strings-x = "" "" x ;
-
- $(equality-assert) : $(empty) ;
- $(not-equality-assert-i) "" : $(empty) ;
- $(not-equality-assert-i) "" "" : $(empty) ;
- $(not-equality-assert-i) : $(empty-strings) ;
- $(not-equality-assert-i) "" : $(empty-strings) ;
- $(equality-assert) "" "" : $(empty-strings) ;
- $(equality-assert) $(empty) : $(empty) ;
- $(equality-assert) $(empty-strings) : $(empty-strings) ;
- $(not-equality-assert-i) $(empty) : $(empty-strings) ;
- $(equality-assert) $(x-empty-strings) : $(x-empty-strings) ;
- $(equality-assert) $(empty-strings-x) : $(empty-strings-x) ;
- $(not-equality-assert) $(empty-strings-x) : $(x-empty-strings) ;
- $(not-equality-assert-i) x : $(x-empty-strings) ;
- $(not-equality-assert) x : $(empty-strings-x) ;
- $(not-equality-assert-i) x : $(x-empty-strings) ;
- $(not-equality-assert-i) x "" : $(x-empty-strings) ;
- $(equality-assert) x "" "" : $(x-empty-strings) ;
- $(not-equality-assert) x : $(empty-strings-x) ;
- $(not-equality-assert) "" x : $(empty-strings-x) ;
- $(equality-assert) "" "" x : $(empty-strings-x) ;
- }
-
-
- # ---------------
- # Equality tests.
- # ---------------
-
- run-equality-test equal : ignore-trailing-empty-strings ;
- run-equality-test exact-equal ;
-
-
- # -------------------------
- # assert.set-equal() tests.
- # -------------------------
-
- set-equal : ;
- not-set-equal "" "" : ;
- set-equal "" "" : "" ;
- set-equal "" "" : "" "" ;
- set-equal a b c : a b c ;
- set-equal a b c : b c a ;
- set-equal a b c a : a b c ;
- set-equal a b c : a b c a ;
- not-set-equal a b c : a b c d ;
- not-set-equal a b c d : a b c ;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/util/container.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/util/container.jam b/ext/kenlm/jam-files/boost-build/util/container.jam
deleted file mode 100644
index dd49639..0000000
--- a/ext/kenlm/jam-files/boost-build/util/container.jam
+++ /dev/null
@@ -1,339 +0,0 @@
-# Copyright 2003 Dave Abrahams
-# Copyright 2002, 2003 Rene Rivera
-# Copyright 2002, 2003, 2004 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-# Various container classes.
-
-# Base for container objects. This lets us construct recursive structures. That
-# is containers with containers in them, specifically so we can tell literal
-# values from node values.
-#
-class node
-{
- rule __init__ (
- value ? # Optional value to set node to initially.
- )
- {
- self.value = $(value) ;
- }
-
- # Set the value of this node, passing nothing will clear it.
- #
- rule set ( value * )
- {
- self.value = $(value) ;
- }
-
- # Get the value of this node.
- #
- rule get ( )
- {
- return $(self.value) ;
- }
-}
-
-
-# A simple vector. Interface mimics the C++ std::vector and std::list, with the
-# exception that indices are one (1) based to follow Jam standard.
-#
-# TODO: Possibly add assertion checks.
-#
-class vector : node
-{
- import numbers ;
- import utility ;
- import sequence ;
-
- rule __init__ (
- values * # Initial contents of vector.
- )
- {
- node.__init__ ;
- self.value = $(values) ;
- }
-
- # Get the value of the first element.
- #
- rule front ( )
- {
- return $(self.value[1]) ;
- }
-
- # Get the value of the last element.
- #
- rule back ( )
- {
- return $(self.value[-1]) ;
- }
-
- # Get the value of the element at the given index, one based. Access to
- # elements of recursive structures is supported directly. Specifying
- # additional index values recursively accesses the elements as containers.
- # For example: [ $(v).at 1 : 2 ] would retrieve the second element of our
- # first element, assuming the first element is a container.
- #
- rule at (
- index # The element index, one based.
- : * # Additional indices to access recursively.
- )
- {
- local r = $(self.value[$(index)]) ;
- if $(2)
- {
- r = [ $(r).at $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ] ;
- }
- return $(r) ;
- }
-
- # Get the value contained in the given element. This has the same
- # functionality and interface as "at" but in addition gets the value of the
- # referenced element, assuming it is a "node".
- #
- rule get-at (
- index # The element index, one based.
- : * # Additional indices to access recursively.
- )
- {
- local r = $(self.value[$(index)]) ;
- if $(2)
- {
- r = [ $(r).at $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ] ;
- }
- return [ $(r).get ] ;
- }
-
- # Insert the given value into the front of the vector pushing the rest of
- # the elements back.
- #
- rule push-front (
- value # Value to become first element.
- )
- {
- self.value = $(value) $(self.value) ;
- }
-
- # Remove the front element from the vector. Does not return the value. No
- # effect if vector is empty.
- #
- rule pop-front ( )
- {
- self.value = $(self.value[2-]) ;
- }
-
- # Add the given value at the end of the vector.
- #
- rule push-back (
- value # Value to become back element.
- )
- {
- self.value += $(value) ;
- }
-
- # Remove the back element from the vector. Does not return the value. No
- # effect if vector is empty.
- #
- rule pop-back ( )
- {
- self.value = $(self.value[1--2]) ;
- }
-
- # Insert the given value at the given index, one based. The values at and to
- # the right of the index are pushed back to make room for the new value.
- # If the index is passed the end of the vector the element is added to the
- # end.
- #
- rule insert (
- index # The index to insert at, one based.
- : value # The value to insert.
- )
- {
- local left = $(self.value[1-$(index)]) ;
- local right = $(self.value[$(index)-]) ;
- if $(right)-is-not-empty
- {
- left = $(left[1--2]) ;
- }
- self.value = $(left) $(value) $(right) ;
- }
-
- # Remove one or more elements from the vector. The range is inclusive, and
- # not specifying an end is equivalent to the [start, start] range.
- #
- rule erase (
- start # Index of first element to remove.
- end ? # Optional, index of last element to remove.
- )
- {
- end ?= $(start) ;
- local left = $(self.value[1-$(start)]) ;
- left = $(left[1--2]) ;
- local right = $(self.value[$(end)-]) ;
- right = $(right[2-]) ;
- self.value = $(left) $(right) ;
- }
-
- # Remove all elements from the vector.
- #
- rule clear ( )
- {
- self.value = ;
- }
-
- # The number of elements in the vector.
- #
- rule size ( )
- {
- return [ sequence.length $(self.value) ] ;
- }
-
- # Returns "true" if there are NO elements in the vector, empty otherwise.
- #
- rule empty ( )
- {
- if ! $(self.value)-is-not-empty
- {
- return true ;
- }
- }
-
- # Returns the textual representation of content.
- #
- rule str ( )
- {
- return "[" [ sequence.transform utility.str : $(self.value) ] "]" ;
- }
-
- # Sorts the vector inplace, calling 'utility.less' for comparisons.
- #
- rule sort ( )
- {
- self.value = [ sequence.insertion-sort $(self.value) : utility.less ] ;
- }
-
- # Returns true if content is equal to the content of other vector. Uses
- # 'utility.equal' for comparison.
- #
- rule equal ( another )
- {
- local mismatch ;
- local size = [ size ] ;
- if $(size) = [ $(another).size ]
- {
- for local i in [ numbers.range 1 $(size) ]
- {
- if ! [ utility.equal [ at $(i) ] [ $(another).at $(i) ] ]
- {
- mismatch = true ;
- }
- }
- }
- else
- {
- mismatch = true ;
- }
-
- if ! $(mismatch)
- {
- return true ;
- }
- }
-}
-
-
-rule __test__ ( )
-{
- import assert ;
- import "class" : new ;
-
- local v1 = [ new vector ] ;
- assert.true $(v1).equal $(v1) ;
- assert.true $(v1).empty ;
- assert.result 0 : $(v1).size ;
- assert.result "[" "]" : $(v1).str ;
- $(v1).push-back b ;
- $(v1).push-front a ;
- assert.result "[" a b "]" : $(v1).str ;
- assert.result a : $(v1).front ;
- assert.result b : $(v1).back ;
- $(v1).insert 2 : d ;
- $(v1).insert 2 : c ;
- $(v1).insert 4 : f ;
- $(v1).insert 4 : e ;
- $(v1).pop-back ;
- assert.result 5 : $(v1).size ;
- assert.result d : $(v1).at 3 ;
- $(v1).pop-front ;
- assert.result c : $(v1).front ;
- assert.false $(v1).empty ;
- $(v1).erase 3 4 ;
- assert.result 2 : $(v1).size ;
-
- local v2 = [ new vector q w e r t y ] ;
- assert.result 6 : $(v2).size ;
- $(v1).push-back $(v2) ;
- assert.result 3 : $(v1).size ;
- local v2-alias = [ $(v1).back ] ;
- assert.result e : $(v2-alias).at 3 ;
- $(v1).clear ;
- assert.true $(v1).empty ;
- assert.false $(v2-alias).empty ;
- $(v2).pop-back ;
- assert.result t : $(v2-alias).back ;
-
- local v3 = [ new vector ] ;
- $(v3).push-back [ new vector 1 2 3 4 5 ] ;
- $(v3).push-back [ new vector a b c ] ;
- assert.result "[" "[" 1 2 3 4 5 "]" "[" a b c "]" "]" : $(v3).str ;
- $(v3).push-back [ new vector [ new vector x y z ] [ new vector 7 8 9 ] ] ;
- assert.result 1 : $(v3).at 1 : 1 ;
- assert.result b : $(v3).at 2 : 2 ;
- assert.result a b c : $(v3).get-at 2 ;
- assert.result 7 8 9 : $(v3).get-at 3 : 2 ;
-
- local v4 = [ new vector 4 3 6 ] ;
- $(v4).sort ;
- assert.result 3 4 6 : $(v4).get ;
- assert.false $(v4).equal $(v3) ;
-
- local v5 = [ new vector 3 4 6 ] ;
- assert.true $(v4).equal $(v5) ;
- # Check that vectors of different sizes are considered non-equal.
- $(v5).pop-back ;
- assert.false $(v4).equal $(v5) ;
-
- local v6 = [ new vector [ new vector 1 2 3 ] ] ;
- assert.true $(v6).equal [ new vector [ new vector 1 2 3 ] ] ;
-
- local v7 = [ new vector 111 222 333 ] ;
- assert.true $(v7).equal $(v7) ;
- $(v7).insert 4 : 444 ;
- assert.result 111 222 333 444 : $(v7).get ;
- $(v7).insert 999 : xxx ;
- assert.result 111 222 333 444 xxx : $(v7).get ;
-
- local v8 = [ new vector "" "" "" ] ;
- assert.true $(v8).equal $(v8) ;
- assert.false $(v8).empty ;
- assert.result 3 : $(v8).size ;
- assert.result "" : $(v8).at 1 ;
- assert.result "" : $(v8).at 2 ;
- assert.result "" : $(v8).at 3 ;
- assert.result : $(v8).at 4 ;
- $(v8).insert 2 : 222 ;
- assert.result 4 : $(v8).size ;
- assert.result "" 222 "" "" : $(v8).get ;
- $(v8).insert 999 : "" ;
- assert.result 5 : $(v8).size ;
- assert.result "" 222 "" "" "" : $(v8).get ;
- $(v8).insert 999 : xxx ;
- assert.result 6 : $(v8).size ;
- assert.result "" 222 "" "" "" xxx : $(v8).get ;
-
- # Regression test for a bug causing vector.equal to compare only the first
- # and the last element in the given vectors.
- local v9 = [ new vector 111 xxx 222 ] ;
- local v10 = [ new vector 111 yyy 222 ] ;
- assert.false $(v9).equal $(v10) ;
-}
[42/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/kernel/errors.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/kernel/errors.jam b/ext/kenlm/jam-files/boost-build/kernel/errors.jam
deleted file mode 100644
index 9563396..0000000
--- a/ext/kenlm/jam-files/boost-build/kernel/errors.jam
+++ /dev/null
@@ -1,287 +0,0 @@
-# Copyright 2003 Dave Abrahams
-# Copyright 2004 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# Print a stack backtrace leading to this rule's caller. Each argument
-# represents a line of output to be printed after the first line of the
-# backtrace.
-#
-rule backtrace ( skip-frames prefix messages * : * )
-{
- local frame-skips = 5 9 13 17 21 25 29 33 37 41 45 49 53 57 61 65 69 73 77 81 ;
- local drop-elements = $(frame-skips[$(skip-frames)]) ;
- if ! ( $(skip-frames) in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 )
- {
- ECHO "warning: backtrace doesn't support skipping $(skip-frames) "
- "frames; using 1 instead." ;
- drop-elements = 5 ;
- }
-
- local args = $(.args) ;
- if $(.user-modules-only)
- {
- local bt = [ nearest-user-location ] ;
- if $(bt)
- {
- ECHO $(prefix) at $(bt) ;
- }
- for local n in $(args)
- {
- if $($(n))-is-defined
- {
- ECHO $(prefix) $($(n)) ;
- }
- }
- }
- else
- {
- # Get the whole backtrace, then drop the initial quadruples
- # corresponding to the frames that must be skipped.
- local bt = [ BACKTRACE ] ;
- bt = $(bt[$(drop-elements)-]) ;
-
- while $(bt)
- {
- local m = [ MATCH ^(.+)\\.$ : $(bt[3]) ] ;
- ECHO $(bt[1]):$(bt[2]): "in" $(bt[4]) "from module" $(m) ;
-
- # The first time through, print each argument on a separate line.
- for local n in $(args)
- {
- if $($(n))-is-defined
- {
- ECHO $(prefix) $($(n)) ;
- }
- }
- args = ; # Kill args so that this never happens again.
-
- # Move on to the next quadruple.
- bt = $(bt[5-]) ;
- }
- }
-}
-
-.args ?= messages 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 ;
-.disabled ?= ;
-.last-error-$(.args) ?= ;
-
-
-# try-catch --
-#
-# This is not really an exception-handling mechanism, but it does allow us to
-# perform some error-checking on our error-checking. Errors are suppressed after
-# a try, and the first one is recorded. Use catch to check that the error
-# message matched expectations.
-
-# Begin looking for error messages.
-#
-rule try ( )
-{
- .disabled += true ;
- .last-error-$(.args) = ;
-}
-
-
-# Stop looking for error messages; generate an error if an argument of messages
-# is not found in the corresponding argument in the error call.
-#
-rule catch ( messages * : * )
-{
- .disabled = $(.disabled[2-]) ; # Pop the stack.
-
- import sequence ;
-
- if ! $(.last-error-$(.args))-is-defined
- {
- error-skip-frames 3 expected an error, but none occurred ;
- }
- else
- {
- for local n in $(.args)
- {
- if ! $($(n)) in $(.last-error-$(n))
- {
- local v = [ sequence.join $($(n)) : " " ] ;
- v ?= "" ;
- local joined = [ sequence.join $(.last-error-$(n)) : " " ] ;
-
- .last-error-$(.args) = ;
- error-skip-frames 3 expected \"$(v)\" in argument $(n) of error
- : got \"$(joined)\" instead ;
- }
- }
- }
-}
-
-
-rule error-skip-frames ( skip-frames messages * : * )
-{
- if ! $(.disabled)
- {
- backtrace $(skip-frames) error: $(messages) : $(2) : $(3) : $(4) : $(5)
- : $(6) : $(7) : $(8) : $(9) : $(10) : $(11) : $(12) : $(13) : $(14)
- : $(15) : $(16) : $(17) : $(18) : $(19) ;
- EXIT ;
- }
- else if ! $(.last-error-$(.args))
- {
- for local n in $(.args)
- {
- # Add an extra empty string so that we always have something in the
- # event of an error.
- .last-error-$(n) = $($(n)) "" ;
- }
- }
-}
-
-if --no-error-backtrace in [ modules.peek : ARGV ]
-{
- .no-error-backtrace = true ;
-}
-
-
-# Print an error message with a stack backtrace and exit.
-#
-rule error ( messages * : * )
-{
- if $(.no-error-backtrace)
- {
- local first-printed ;
- # Print each argument on a separate line.
- for local n in $(.args)
- {
- if $($(n))-is-defined
- {
- if ! $(first-printed)
- {
- ECHO error: $($(n)) ;
- first-printed = true ;
- }
- else
- {
- ECHO $($(n)) ;
- }
- }
- }
- EXIT ;
- }
- else
- {
- error-skip-frames 3 $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) :
- $(8) : $(9) : $(10) : $(11) : $(12) : $(13) : $(14) : $(15) : $(16)
- : $(17) : $(18) : $(19) ;
- }
-}
-
-
-# Same as 'error', but the generated backtrace will include only user files.
-#
-rule user-error ( messages * : * )
-{
- .user-modules-only = 1 ;
- error-skip-frames 3 $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) :
- $(9) : $(10) : $(11) : $(12) : $(13) : $(14) : $(15) : $(16) : $(17) :
- $(18) : $(19) ;
-}
-
-
-# Print a warning message with a stack backtrace and exit.
-#
-rule warning
-{
- backtrace 2 warning: $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) :
- $(9) : $(10) : $(11) : $(12) : $(13) : $(14) : $(15) : $(16) : $(17) :
- $(18) : $(19) ;
-}
-
-
-# Convert an arbitrary argument list into a list with ":" separators and quoted
-# elements representing the same information. This is mostly useful for
-# formatting descriptions of arguments with which a rule was called when
-# reporting an error.
-#
-rule lol->list ( * )
-{
- local result ;
- local remaining = 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 ;
- while $($(remaining))
- {
- local n = $(remaining[1]) ;
- remaining = $(remaining[2-]) ;
-
- if $(n) != 1
- {
- result += ":" ;
- }
- result += \"$($(n))\" ;
- }
- return $(result) ;
-}
-
-
-# Return the file:line for the nearest entry in backtrace which correspond to a
-# user module.
-#
-rule nearest-user-location ( )
-{
- local bt = [ BACKTRACE ] ;
-
- local result ;
- while $(bt) && ! $(result)
- {
- local m = [ MATCH ^(.+)\\.$ : $(bt[3]) ] ;
- local user-modules = ([Jj]amroot(.jam|.v2|)|([Jj]amfile(.jam|.v2|)|user-config.jam|site-config.jam|project-config.jam|project-root.jam) ;
-
- if [ MATCH $(user-modules) : $(bt[1]:D=) ]
- {
- result = $(bt[1]):$(bt[2]) ;
- }
- bt = $(bt[5-]) ;
- }
- return $(result) ;
-}
-
-
-# If optimized rule is available in Jam, use it.
-if NEAREST_USER_LOCATION in [ RULENAMES ]
-{
- rule nearest-user-location ( )
- {
- local r = [ NEAREST_USER_LOCATION ] ;
- return $(r[1]):$(r[2]) ;
- }
-}
-
-
-rule __test__ ( )
-{
- # Show that we can correctly catch an expected error.
- try ;
- {
- error an error occurred : somewhere ;
- }
- catch an error occurred : somewhere ;
-
- # Show that unexpected errors generate real errors.
- try ;
- {
- try ;
- {
- error an error occurred : somewhere ;
- }
- catch an error occurred : nowhere ;
- }
- catch expected \"nowhere\" in argument 2 ;
-
- # Show that not catching an error where one was expected is an error.
- try ;
- {
- try ;
- {
- }
- catch ;
- }
- catch expected an error, but none occurred ;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/kernel/modules.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/kernel/modules.jam b/ext/kenlm/jam-files/boost-build/kernel/modules.jam
deleted file mode 100644
index 4258225..0000000
--- a/ext/kenlm/jam-files/boost-build/kernel/modules.jam
+++ /dev/null
@@ -1,359 +0,0 @@
-# Copyright 2003 Dave Abrahams
-# Copyright 2003, 2005 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# Essentially an include guard; ensures that no module is loaded multiple times.
-.loaded ?= ;
-
-# A list of modules currently being loaded for error reporting of circular
-# dependencies.
-.loading ?= ;
-
-# A list of modules needing to be tested using their __test__ rule.
-.untested ?= ;
-
-# A list of modules which have been tested using their __test__ rule.
-.tested ?= ;
-
-
-# Runs internal Boost Build unit tests for the specified module. The module's
-# __test__ rule is executed in its own module to eliminate any inadvertent
-# effects of testing module dependencies (such as assert) on the module itself.
-#
-local rule run-module-test ( m )
-{
- local tested-modules = [ modules.peek modules : .tested ] ;
-
- if ( ! $(m) in $(tested-modules) ) # Avoid recursive test invocations.
- && ( ( --debug in $(argv) ) || ( --debug-module=$(m) in $(argv) ) )
- {
- modules.poke modules : .tested : $(tested-modules) $(m) ;
-
- if ! ( __test__ in [ RULENAMES $(m) ] )
- {
- local argv = [ peek : ARGV ] ;
- if ! ( --quiet in $(argv) ) && ( --debug-tests in $(argv) )
- {
- ECHO warning: no __test__ rule defined in module $(m) ;
- }
- }
- else
- {
- if ! ( --quiet in $(argv) )
- {
- ECHO testing module $(m)... ;
- }
-
- local test-module = __test-$(m)__ ;
- IMPORT $(m) : [ RULENAMES $(m) ] : $(test-module) : [ RULENAMES $(m)
- ] ;
- IMPORT $(m) : __test__ : $(test-module) : __test__ : LOCALIZE ;
- module $(test-module)
- {
- __test__ ;
- }
- }
- }
-}
-
-
-# Return the binding of the given module.
-#
-rule binding ( module )
-{
- return $($(module).__binding__) ;
-}
-
-
-# Sets the module-local value of a variable. This is the most reliable way to
-# set a module-local variable in a different module; it eliminates issues of
-# name shadowing due to dynamic scoping.
-#
-rule poke ( module-name ? : variables + : value * )
-{
- module $(<)
- {
- $(>) = $(3) ;
- }
-}
-
-
-# Returns the module-local value of a variable. This is the most reliable way to
-# examine a module-local variable in a different module; it eliminates issues of
-# name shadowing due to dynamic scoping.
-#
-rule peek ( module-name ? : variables + )
-{
- module $(<)
- {
- return $($(>)) ;
- }
-}
-
-
-# Call the given rule locally in the given module. Use this for rules accepting
-# rule names as arguments, so that the passed rule may be invoked in the context
-# of the rule's caller (for example, if the rule accesses module globals or is a
-# local rule). Note that rules called this way may accept at most 18 parameters.
-#
-rule call-in ( module-name ? : rule-name args * : * )
-{
- module $(module-name)
- {
- return [ $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) : $(10) :
- $(11) : $(12) : $(13) : $(14) : $(15) : $(16) : $(17) : $(18) :
- $(19) ] ;
- }
-}
-
-
-# Given a possibly qualified rule name and arguments, remove any initial module
-# qualification from the rule and invoke it in that module. If there is no
-# module qualification, the rule is invoked in the global module. Note that
-# rules called this way may accept at most 18 parameters.
-#
-rule call-locally ( qualified-rule-name args * : * )
-{
- local module-rule = [ MATCH (.*)\\.(.*) : $(qualified-rule-name) ] ;
- local rule-name = $(module-rule[2]) ;
- rule-name ?= $(qualified-rule-name) ;
- # We pass only 18 parameters here since Boost Jam allows at most 19 rule
- # parameter positions and the call-in rule already uses up the initial
- # position for the module name.
- return [ call-in $(module-rule[1]) : $(rule-name) $(args) : $(2) : $(3) :
- $(4) : $(5) : $(6) : $(7) : $(8) : $(9) : $(10) : $(11) : $(12) : $(13)
- $(14) : $(15) : $(16) : $(17) : $(18) : $(19) ] ;
-}
-
-
-# Load the indicated module if it is not already loaded.
-#
-rule load (
- module-name # Name of module to load. Rules will be defined in this
- # module.
- : filename ? # (partial) path to file; Defaults to $(module-name).jam.
- : search * # Directories in which to search for filename. Defaults to
- # $(BOOST_BUILD_PATH).
-)
-{
- # Avoid loading modules twice.
- if ! ( $(module-name) in $(.loaded) )
- {
- filename ?= $(module-name).jam ;
-
- # Mark the module loaded so we do not try to load it recursively.
- .loaded += $(module-name) ;
-
- # Suppress tests if any module loads are already in progress.
- local suppress-test = $(.loading[1]) ;
-
- # Push this module on the loading stack.
- .loading += $(module-name) ;
-
- # Remember that it is untested.
- .untested += $(module-name) ;
-
- # Insert the new module's __name__ and __file__ globals.
- poke $(module-name) : __name__ : $(module-name) ;
- poke $(module-name) : __file__ : $(filename) ;
-
- module $(module-name)
- {
- # Add some grist so that the module will have a unique target name.
- local module-target = $(__file__:G=module@) ;
-
- local search = $(3) ;
- search ?= [ modules.peek : BOOST_BUILD_PATH ] ;
- SEARCH on $(module-target) = $(search) ;
- BINDRULE on $(module-target) = modules.record-binding ;
-
- include $(module-target) ;
-
- # Allow the module to see its own names with full qualification.
- local rules = [ RULENAMES $(__name__) ] ;
- IMPORT $(__name__) : $(rules) : $(__name__) : $(__name__).$(rules) ;
- }
-
- if $(module-name) != modules && ! [ binding $(module-name) ]
- {
- import errors ;
- errors.error "Could not find module" $(module-name) in $(search) ;
- }
-
- # Pop the loading stack. Must happen before testing or we will run into
- # a circular loading dependency.
- .loading = $(.loading[1--2]) ;
-
- # Run any pending tests if this is an outer load.
- if ! $(suppress-test)
- {
- local argv = [ peek : ARGV ] ;
- for local m in $(.untested)
- {
- run-module-test $(m) ;
- }
- .untested = ;
- }
- }
- else if $(module-name) in $(.loading)
- {
- import errors ;
- errors.error loading \"$(module-name)\"
- : circular module loading dependency:
- : $(.loading)" ->" $(module-name) ;
- }
-}
-
-
-# This helper is used by load (above) to record the binding (path) of each
-# loaded module.
-#
-rule record-binding ( module-target : binding )
-{
- $(.loading[-1]).__binding__ = $(binding) ;
-}
-
-
-# Transform each path in the list, with all backslashes converted to forward
-# slashes and all detectable redundancy removed. Something like this is probably
-# needed in path.jam, but I am not sure of that, I do not understand it, and I
-# am not ready to move all of path.jam into the kernel.
-#
-local rule normalize-raw-paths ( paths * )
-{
- local result ;
- for p in $(paths:T)
- {
- result += [ NORMALIZE_PATH $(p) ] ;
- }
- return $(result) ;
-}
-
-
-.cwd = [ PWD ] ;
-
-
-# Load the indicated module and import rule names into the current module. Any
-# members of rules-opt will be available without qualification in the caller's
-# module. Any members of rename-opt will be taken as the names of the rules in
-# the caller's module, in place of the names they have in the imported module.
-# If rules-opt = '*', all rules from the indicated module are imported into the
-# caller's module. If rename-opt is supplied, it must have the same number of
-# elements as rules-opt.
-#
-rule import ( module-names + : rules-opt * : rename-opt * )
-{
- if ( $(rules-opt) = * || ! $(rules-opt) ) && $(rename-opt)
- {
- import errors ;
- errors.error "Rule aliasing is only available for explicit imports." ;
- }
-
- if $(module-names[2]) && ( $(rules-opt) || $(rename-opt) )
- {
- import errors ;
- errors.error "When loading multiple modules, no specific rules or"
- "renaming is allowed" ;
- }
-
- local caller = [ CALLER_MODULE ] ;
-
- # Import each specified module
- for local m in $(module-names)
- {
- if ! $(m) in $(.loaded)
- {
- # If the importing module is not already in the BOOST_BUILD_PATH,
- # prepend it to the path. We do not want to invert the search order
- # of modules that are already there.
-
- local caller-location ;
- if $(caller)
- {
- caller-location = [ binding $(caller) ] ;
- caller-location = $(caller-location:D) ;
- caller-location = [ normalize-raw-paths
- $(caller-location:R=$(.cwd)) ] ;
- }
-
- local search = [ peek : BOOST_BUILD_PATH ] ;
- search = [ normalize-raw-paths $(search:R=$(.cwd)) ] ;
-
- if $(caller-location) && ! $(caller-location) in $(search)
- {
- search = $(caller-location) $(search) ;
- }
-
- load $(m) : : $(search) ;
- }
-
- IMPORT_MODULE $(m) : $(caller) ;
-
- if $(rules-opt)
- {
- local source-names ;
- if $(rules-opt) = *
- {
- local all-rules = [ RULENAMES $(m) ] ;
- source-names = $(all-rules) ;
- }
- else
- {
- source-names = $(rules-opt) ;
- }
- local target-names = $(rename-opt) ;
- target-names ?= $(source-names) ;
- IMPORT $(m) : $(source-names) : $(caller) : $(target-names) ;
- }
- }
-}
-
-
-# Define exported copies in $(target-module) of all rules exported from
-# $(source-module). Also make them available in the global module with
-# qualification, so that it is just as though the rules were defined originally
-# in $(target-module).
-#
-rule clone-rules ( source-module target-module )
-{
- local r = [ RULENAMES $(source-module) ] ;
- IMPORT $(source-module) : $(r) : $(target-module) : $(r) : LOCALIZE ;
- EXPORT $(target-module) : $(r) ;
- IMPORT $(target-module) : $(r) : : $(target-module).$(r) ;
-}
-
-
-# These rules need to be available in all modules to implement module loading
-# itself and other fundamental operations.
-local globalize = peek poke record-binding ;
-IMPORT modules : $(globalize) : : modules.$(globalize) ;
-
-
-rule __test__ ( )
-{
- import assert ;
- import modules : normalize-raw-paths ;
-
- module modules.__test__
- {
- foo = bar ;
- }
-
- assert.result bar : peek modules.__test__ : foo ;
-
- poke modules.__test__ : foo : bar baz ;
- assert.result bar baz : peek modules.__test__ : foo ;
-
- assert.result c:/foo/bar : normalize-raw-paths c:/x/../foo/./xx/yy/../../bar ;
- assert.result . : normalize-raw-paths . ;
- assert.result .. : normalize-raw-paths .. ;
- assert.result ../.. : normalize-raw-paths ../.. ;
- assert.result .. : normalize-raw-paths ./.. ;
- assert.result / / : normalize-raw-paths / \\ ;
- assert.result a : normalize-raw-paths a ;
- assert.result a : normalize-raw-paths a/ ;
- assert.result /a : normalize-raw-paths /a/ ;
- assert.result / : normalize-raw-paths /a/.. ;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/options/help.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/options/help.jam b/ext/kenlm/jam-files/boost-build/options/help.jam
deleted file mode 100644
index b507e1e..0000000
--- a/ext/kenlm/jam-files/boost-build/options/help.jam
+++ /dev/null
@@ -1,212 +0,0 @@
-# Copyright 2003 Dave Abrahams
-# Copyright 2003, 2006 Rene Rivera
-# Copyright 2003, 2006 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-# This module is the plug-in handler for the --help and --help-.*
-# command-line options
-import modules ;
-import assert ;
-import doc : do-scan set-option set-output set-output-file print-help-usage print-help-top ;
-import sequence ;
-import set ;
-import project ;
-import print ;
-import os ;
-import version ;
-import path ;
-
-# List of possible modules, but which really aren't.
-#
-.not-modules =
- boost-build bootstrap site-config test user-config
- -tools allyourbase boost-base features python stlport testing unit-tests ;
-
-# The help system options are parsed here and handed off to the doc
-# module to translate into documentation requests and actions. The
-# understood options are:
-#
-# --help-disable-<option>
-# --help-doc-options
-# --help-enable-<option>
-# --help-internal
-# --help-options
-# --help-usage
-# --help-output <type>
-# --help-output-file <file>
-# --help [<module-or-class>]
-#
-rule process (
- command # The option.
- : values * # The values, starting after the "=".
- )
-{
- assert.result --help : MATCH ^(--help).* : $(command) ;
- local did-help = ;
- switch $(command)
- {
- case --help-internal :
- local path-to-modules = [ modules.peek : BOOST_BUILD_PATH ] ;
- path-to-modules ?= . ;
- local possible-modules = [ GLOB $(path-to-modules) : *\\.jam ] ;
- local not-modules = [ GLOB $(path-to-modules) : *$(.not-modules)\\.jam ] ;
- local modules-to-list =
- [ sequence.insertion-sort
- [ set.difference $(possible-modules:D=:S=) : $(not-modules:D=:S=) ] ] ;
- local modules-to-scan ;
- for local m in $(modules-to-list)
- {
- local module-files = [ GLOB $(path-to-modules) : $(m)\\.jam ] ;
- modules-to-scan += $(module-files[1]) ;
- }
- do-scan $(modules-to-scan) : print-help-all ;
- did-help = true ;
-
- case --help-enable-* :
- local option = [ MATCH --help-enable-(.*) : $(command) ] ; option = $(option:L) ;
- set-option $(option) : enabled ;
- did-help = true ;
-
- case --help-disable-* :
- local option = [ MATCH --help-disable-(.*) : $(command) ] ; option = $(option:L) ;
- set-option $(option) ;
- did-help = true ;
-
- case --help-output :
- set-output $(values[1]) ;
- did-help = true ;
-
- case --help-output-file :
- set-output-file $(values[1]) ;
- did-help = true ;
-
- case --help-doc-options :
- local doc-module-spec = [ split-symbol doc ] ;
- do-scan $(doc-module-spec[1]) : print-help-options ;
- did-help = true ;
-
- case --help-options :
- print-help-usage ;
- did-help = true ;
-
- case --help :
- local spec = $(values[1]) ;
- if $(spec)
- {
- local spec-parts = [ split-symbol $(spec) ] ;
- if $(spec-parts)
- {
- if $(spec-parts[2])
- {
- do-scan $(spec-parts[1]) : print-help-classes $(spec-parts[2]) ;
- do-scan $(spec-parts[1]) : print-help-rules $(spec-parts[2]) ;
- do-scan $(spec-parts[1]) : print-help-variables $(spec-parts[2]) ;
- }
- else
- {
- do-scan $(spec-parts[1]) : print-help-module ;
- }
- }
- else
- {
- EXIT "Unrecognized help option '"$(command)" "$(spec)"'." ;
- }
- }
- else
- {
- version.print ;
- ECHO ;
- # First print documentation from the current Jamfile, if any.
- # FIXME: Generally, this duplication of project.jam logic is bad.
- local names = [ modules.peek project : JAMROOT ]
- [ modules.peek project : JAMFILE ] ;
- local project-file = [ path.glob . : $(names) ] ;
- if ! $(project-file)
- {
- project-file = [ path.glob-in-parents . : $(names) ] ;
- }
-
- for local p in $(project-file)
- {
- do-scan $(p) : print-help-project $(p) ;
- }
-
- # Next any user-config help.
- local user-path = [ os.home-directories ] [ os.environ BOOST_BUILD_PATH ] ;
- local user-config = [ GLOB $(user-path) : user-config.jam ] ;
- if $(user-config)
- {
- do-scan $(user-config[1]) : print-help-config user $(user-config[1]) ;
- }
-
- # Next any site-config help.
- local site-config = [ GLOB $(user-path) : site-config.jam ] ;
- if $(site-config)
- {
- do-scan $(site-config[1]) : print-help-config site $(site-config[1]) ;
- }
-
- # Then the overall help.
- print-help-top ;
- }
- did-help = true ;
- }
- if $(did-help)
- {
- UPDATE all ;
- NOCARE all ;
- }
- return $(did-help) ;
-}
-
-# Split a reference to a symbol into module and symbol parts.
-#
-local rule split-symbol (
- symbol # The symbol to split.
- )
-{
- local path-to-modules = [ modules.peek : BOOST_BUILD_PATH ] ;
- path-to-modules ?= . ;
- local module-name = $(symbol) ;
- local symbol-name = ;
- local result = ;
- while ! $(result)
- {
- local module-path = [ GLOB $(path-to-modules) : $(module-name)\\.jam ] ;
- if $(module-path)
- {
- # The 'module-name' in fact refers to module. Return the full
- # module path and a symbol within it. If 'symbol' passed to this
- # rule is already module, 'symbol-name' will be empty. Otherwise,
- # it's initialized on the previous loop iteration.
- # In case there are several modules by this name,
- # use the first one.
- result = $(module-path[1]) $(symbol-name) ;
- }
- else
- {
- if ! $(module-name:S)
- {
- result = - ;
- }
- else
- {
- local next-symbol-part = [ MATCH ^.(.*) : $(module-name:S) ] ;
- if $(symbol-name)
- {
- symbol-name = $(next-symbol-part).$(symbol-name) ;
- }
- else
- {
- symbol-name = $(next-symbol-part) ;
- }
- module-name = $(module-name:B) ;
- }
- }
- }
- if $(result) != -
- {
- return $(result) ;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/site-config.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/site-config.jam b/ext/kenlm/jam-files/boost-build/site-config.jam
deleted file mode 100644
index 6afe526..0000000
--- a/ext/kenlm/jam-files/boost-build/site-config.jam
+++ /dev/null
@@ -1,11 +0,0 @@
-# Copyright 1999-2012 Gentoo Foundation
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-# Define two new variants to be used when building boost (or separate boost-libs)
-# on Gentoo. The two variants make use of Gentoo-specific optimization and debug-symbols
-# values "none" which are not part of the official boost distribution.
-# DO NOT RELY ON THE FOLLOWING VARIANTS TO BE PRESENT ON OTHER OS!
-variant gentoorelease : release : <optimization>none <debug-symbols>none <runtime-link>shared ;
-variant gentoodebug : debug : <optimization>none <debug-symbols>on <runtime-link>shared ;
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/acc.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/acc.jam b/ext/kenlm/jam-files/boost-build/tools/acc.jam
deleted file mode 100644
index f04c9dc..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/acc.jam
+++ /dev/null
@@ -1,118 +0,0 @@
-# Copyright Vladimir Prus 2004.
-# Copyright Toon Knapen 2004.
-# Copyright Boris Gubenko 2007.
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt
-# or copy at http://www.boost.org/LICENSE_1_0.txt)
-
-#
-# Boost.Build V2 toolset for the HP aC++ compiler.
-#
-
-import toolset : flags ;
-import feature ;
-import generators ;
-import common ;
-
-feature.extend toolset : acc ;
-toolset.inherit acc : unix ;
-generators.override builtin.lib-generator : acc.prebuilt ;
-generators.override acc.searched-lib-generator : searched-lib-generator ;
-
-# Configures the acc toolset.
-rule init ( version ? : user-provided-command * : options * )
-{
- local condition = [ common.check-init-parameters acc
- : version $(version) ] ;
-
- local command = [ common.get-invocation-command acc : aCC
- : $(user-provided-command) ] ;
-
- common.handle-options acc : $(condition) : $(command) : $(options) ;
-}
-
-
-# Declare generators
-generators.register-c-compiler acc.compile.c : C : OBJ : <toolset>acc ;
-generators.register-c-compiler acc.compile.c++ : CPP : OBJ : <toolset>acc ;
-
-# Declare flags.
-flags acc CFLAGS <optimization>off : ;
-flags acc CFLAGS <optimization>speed : -O3 ;
-flags acc CFLAGS <optimization>space : -O2 ;
-
-flags acc CFLAGS <inlining>off : +d ;
-flags acc CFLAGS <inlining>on : ;
-flags acc CFLAGS <inlining>full : ;
-
-flags acc C++FLAGS <exception-handling>off : ;
-flags acc C++FLAGS <exception-handling>on : ;
-
-flags acc C++FLAGS <rtti>off : ;
-flags acc C++FLAGS <rtti>on : ;
-
-# We want the full path to the sources in the debug symbols because otherwise
-# the debugger won't find the sources when we use boost.build.
-flags acc CFLAGS <debug-symbols>on : -g ;
-flags acc LINKFLAGS <debug-symbols>on : -g ;
-flags acc LINKFLAGS <debug-symbols>off : -s ;
-
-# V2 does not have <shared-linkable>, not sure what this meant in V1.
-# flags acc CFLAGS <shared-linkable>true : +Z ;
-
-flags acc CFLAGS <profiling>on : -pg ;
-flags acc LINKFLAGS <profiling>on : -pg ;
-
-flags acc CFLAGS <address-model>64 : +DD64 ;
-flags acc LINKFLAGS <address-model>64 : +DD64 ;
-
-# It is unknown if there's separate option for rpath used only
-# at link time, similar to -rpath-link in GNU. We'll use -L.
-flags acc RPATH_LINK : <xdll-path> ;
-
-flags acc CFLAGS <cflags> ;
-flags acc C++FLAGS <cxxflags> ;
-flags acc DEFINES <define> ;
-flags acc UNDEFS <undef> ;
-flags acc HDRS <include> ;
-flags acc STDHDRS <sysinclude> ;
-flags acc LINKFLAGS <linkflags> ;
-flags acc ARFLAGS <arflags> ;
-
-flags acc LIBPATH <library-path> ;
-flags acc NEEDLIBS <library-file> ;
-flags acc FINDLIBS <find-shared-library> ;
-flags acc FINDLIBS <find-static-library> ;
-
-# Select the compiler name according to the threading model.
-flags acc CFLAGS <threading>multi : -mt ;
-flags acc LINKFLAGS <threading>multi : -mt ;
-
-flags acc.compile.c++ TEMPLATE_DEPTH <c++-template-depth> ;
-
-
-actions acc.link bind NEEDLIBS
-{
- $(CONFIG_COMMAND) -AA $(LINKFLAGS) -o "$(<[1])" -L"$(RPATH_LINK)" -L$(LIBPATH) -L$(STDLIBPATH) "$(>)" "$(NEEDLIBS)" "$(NEEDLIBS)" -l$(FINDLIBS) $(OPTIONS)
-}
-
-SPACE = " " ;
-actions acc.link.dll bind NEEDLIBS
-{
- $(CONFIG_COMMAND) -AA -b $(LINKFLAGS) -o "$(<[1])" -L"$(RPATH_LINK)" -Wl,+h$(<[-1]:D=) -L$(LIBPATH) -L$(STDLIBPATH) "$(>)" "$(NEEDLIBS)" "$(NEEDLIBS)" -l$(FINDLIBS) $(OPTIONS)
-}
-
-actions acc.compile.c
-{
- cc -c -I$(BOOST_ROOT) -U$(UNDEFS) -D$(DEFINES) $(CFLAGS) -I"$(HDRS)" -I"$(STDHDRS)" -o "$(<)" "$(>)" $(OPTIONS)
-}
-
-actions acc.compile.c++
-{
- $(CONFIG_COMMAND) -AA -c -Wc,--pending_instantiations=$(TEMPLATE_DEPTH) -I$(BOOST_ROOT) -U$(UNDEFS) -D$(DEFINES) $(CFLAGS) $(C++FLAGS) -I"$(HDRS)" -I"$(STDHDRS)" -o "$(<)" "$(>)" $(OPTIONS)
-}
-
-actions updated together piecemeal acc.archive
-{
- ar ru$(ARFLAGS:E="") "$(<)" "$(>)"
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/auto-index.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/auto-index.jam b/ext/kenlm/jam-files/boost-build/tools/auto-index.jam
deleted file mode 100644
index 41d0482..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/auto-index.jam
+++ /dev/null
@@ -1,204 +0,0 @@
-
-import feature ;
-import generators ;
-import "class" ;
-import toolset ;
-import targets ;
-import "class" : new ;
-import project ;
-
-feature.feature auto-index : off "on" ;
-feature.feature auto-index-internal : off "on" ;
-feature.feature auto-index-verbose : off "on" ;
-feature.feature auto-index-no-duplicates : off "on" ;
-feature.feature auto-index-script : : free path ;
-feature.feature auto-index-prefix : : free path ;
-feature.feature auto-index-type : : free ;
-feature.feature auto-index-section-names : "on" off ;
-
-toolset.flags auto-index.auto-index FLAGS <auto-index-internal>on : --internal-index ;
-toolset.flags auto-index.auto-index SCRIPT <auto-index-script> ;
-toolset.flags auto-index.auto-index PREFIX <auto-index-prefix> ;
-toolset.flags auto-index.auto-index INDEX_TYPE <auto-index-type> ;
-toolset.flags auto-index.auto-index FLAGS <auto-index-verbose>on : --verbose ;
-toolset.flags auto-index.auto-index FLAGS <auto-index-no-duplicates>on : --no-duplicates ;
-toolset.flags auto-index.auto-index FLAGS <auto-index-section-names>off : --no-section-names ;
-
-# <auto-index-binary> shell command to run AutoIndex
-# <auto-index-binary-dependencies> targets to build AutoIndex from sources.
-feature.feature <auto-index-binary> : : free ;
-feature.feature <auto-index-binary-dependencies> : : free dependency ;
-
-class auto-index-generator : generator
-{
- import common modules path targets build-system ;
- rule run ( project name ? : property-set : sources * )
- {
- # AutoIndex invocation command and dependencies.
- local auto-index-binary = [ modules.peek auto-index : .command ] ;
- local auto-index-binary-dependencies ;
-
- if $(auto-index-binary)
- {
- # Use user-supplied command.
- auto-index-binary = [ common.get-invocation-command auto-index : auto-index : $(auto-index-binary) ] ;
- }
- else
- {
- # Search for AutoIndex sources in sensible places, like
- # $(BOOST_ROOT)/tools/auto_index
- # $(BOOST_BUILD_PATH)/../../auto_index
-
- # And build auto-index executable from sources.
-
- local boost-root = [ modules.peek : BOOST_ROOT ] ;
- local boost-build-path = [ build-system.location ] ;
- local boost-build-path2 = [ modules.peek : BOOST_BUILD_PATH ] ;
-
- local auto-index-dir ;
-
- if $(boost-root)
- {
- auto-index-dir += [ path.join $(boost-root) tools ] ;
- }
-
- if $(boost-build-path)
- {
- auto-index-dir += $(boost-build-path)/../.. ;
- }
- if $(boost-build-path2)
- {
- auto-index-dir += $(boost-build-path2)/.. ;
- }
-
- #ECHO $(auto-index-dir) ;
- auto-index-dir = [ path.glob $(auto-index-dir) : auto_index ] ;
- #ECHO $(auto-index-dir) ;
-
- # If the AutoIndex source directory was found, mark its main target
- # as a dependency for the current project. Otherwise, try to find
- # 'auto-index' in user's PATH
- if $(auto-index-dir)
- {
- auto-index-dir = [ path.make $(auto-index-dir[1]) ] ;
- auto-index-dir = $(auto-index-dir)/build ;
-
- #ECHO $(auto-index-dir) ;
-
- # Get the main-target in AutoIndex directory.
- local auto-index-main-target = [ targets.resolve-reference $(auto-index-dir) : $(project) ] ;
-
- #ECHO $(auto-index-main-target) ;
-
- # The first element are actual targets, the second are
- # properties found in target-id. We do not care about these
- # since we have passed the id ourselves.
- auto-index-main-target =
- [ $(auto-index-main-target[1]).main-target auto_index ] ;
-
- #ECHO $(auto-index-main-target) ;
-
- auto-index-binary-dependencies =
- [ $(auto-index-main-target).generate [ $(property-set).propagated ] ] ;
-
- # Ignore usage-requirements returned as first element.
- auto-index-binary-dependencies = $(auto-index-binary-dependencies[2-]) ;
-
- # Some toolsets generate extra targets (e.g. RSP). We must mark
- # all targets as dependencies for the project, but we will only
- # use the EXE target for auto-index-to-boostbook translation.
- for local target in $(auto-index-binary-dependencies)
- {
- if [ $(target).type ] = EXE
- {
- auto-index-binary =
- [ path.native
- [ path.join
- [ $(target).path ]
- [ $(target).name ]
- ]
- ] ;
- }
- }
- }
- else
- {
- ECHO "AutoIndex warning: The path to the auto-index executable was" ;
- ECHO " not provided. Additionally, couldn't find AutoIndex" ;
- ECHO " sources searching in" ;
- ECHO " * BOOST_ROOT/tools/auto-index" ;
- ECHO " * BOOST_BUILD_PATH/../../auto-index" ;
- ECHO " Will now try to find a precompiled executable by searching" ;
- ECHO " the PATH for 'auto-index'." ;
- ECHO " To disable this warning in the future, or to completely" ;
- ECHO " avoid compilation of auto-index, you can explicitly set the" ;
- ECHO " path to a auto-index executable command in user-config.jam" ;
- ECHO " or site-config.jam with the call" ;
- ECHO " using auto-index : /path/to/auto-index ;" ;
-
- # As a last resort, search for 'auto-index' command in path. Note
- # that even if the 'auto-index' command is not found,
- # get-invocation-command will still return 'auto-index' and might
- # generate an error while generating the virtual-target.
-
- auto-index-binary = [ common.get-invocation-command auto-index : auto-index ] ;
- }
- }
-
- # Add $(auto-index-binary-dependencies) as a dependency of the current
- # project and set it as the <auto-index-binary> feature for the
- # auto-index-to-boostbook rule, below.
- property-set = [ $(property-set).add-raw
- <dependency>$(auto-index-binary-dependencies)
- <auto-index-binary>$(auto-index-binary)
- <auto-index-binary-dependencies>$(auto-index-binary-dependencies)
- ] ;
-
- #ECHO "binary = " $(auto-index-binary) ;
- #ECHO "dependencies = " $(auto-index-binary-dependencies) ;
-
- return [ generator.run $(project) $(name) : $(property-set) : $(sources) ] ;
- }
-}
-
-# Initialization of toolset.
-#
-# Parameters:
-# command ? -> path to AutoIndex executable.
-#
-# When command is not supplied toolset will search for AutoIndex directory and
-# compile the executable from source. If that fails we still search the path for
-# 'auto_index'.
-#
-rule init (
- command ? # path to the AutoIndex executable.
- )
-{
- if ! $(.initialized)
- {
- .initialized = true ;
- .command = $(command) ;
- }
-}
-
-toolset.flags auto-index.auto-index AI-COMMAND <auto-index-binary> ;
-toolset.flags auto-index.auto-index AI-DEPENDENCIES <auto-index-binary-dependencies> ;
-
-generators.register [ class.new auto-index-generator auto-index.auto-index : DOCBOOK : DOCBOOK(%.auto_index) : <auto-index>on ] ;
-generators.override auto-index.auto-index : boostbook.boostbook-to-docbook ;
-
-rule auto-index ( target : source : properties * )
-{
- # Signal dependency of auto-index sources on <auto-index-binary-dependencies>
- # upon invocation of auto-index-to-boostbook.
- #ECHO "AI-COMMAND= " $(AI-COMMAND) ;
- DEPENDS $(target) : [ on $(target) return $(AI-DEPENDENCIES) ] ;
- #DEPENDS $(target) : [ on $(target) return $(SCRIPT) ] ;
-}
-
-actions auto-index
-{
- $(AI-COMMAND) $(FLAGS) "--prefix="$(PREFIX) "--script="$(SCRIPT) "--index-type="$(INDEX_TYPE) "--in="$(>) "--out="$(<)
-}
-
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/bison.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/bison.jam b/ext/kenlm/jam-files/boost-build/tools/bison.jam
deleted file mode 100644
index 0689d4b..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/bison.jam
+++ /dev/null
@@ -1,32 +0,0 @@
-# Copyright 2003 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-import generators ;
-import feature ;
-import type ;
-import property ;
-
-feature.feature bison.prefix : : free ;
-type.register Y : y ;
-type.register YY : yy ;
-generators.register-standard bison.bison : Y : C H ;
-generators.register-standard bison.bison : YY : CPP HPP ;
-
-rule init ( )
-{
-}
-
-rule bison ( dst dst_header : src : properties * )
-{
- local r = [ property.select bison.prefix : $(properties) ] ;
- if $(r)
- {
- PREFIX_OPT on $(<) = -p $(r:G=) ;
- }
-}
-
-actions bison
-{
- bison $(PREFIX_OPT) -d -o $(<[1]) $(>)
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/boostbook-config.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/boostbook-config.jam b/ext/kenlm/jam-files/boost-build/tools/boostbook-config.jam
deleted file mode 100644
index 6e3f3dd..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/boostbook-config.jam
+++ /dev/null
@@ -1,13 +0,0 @@
-#~ Copyright 2005 Rene Rivera.
-#~ Distributed under the Boost Software License, Version 1.0.
-#~ (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-# Automatic configuration for BoostBook tools. To use, just import this module.
-#
-# This module is deprecated.
-# using boostbook ;
-# with no arguments now suffices.
-
-import toolset : using ;
-
-using boostbook ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/boostbook.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/boostbook.jam b/ext/kenlm/jam-files/boost-build/tools/boostbook.jam
deleted file mode 100644
index de83dec..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/boostbook.jam
+++ /dev/null
@@ -1,773 +0,0 @@
-# Copyright 2003, 2004, 2005 Dave Abrahams
-# Copyright 2003, 2004, 2005 Douglas Gregor
-# Copyright 2005, 2006, 2007 Rene Rivera
-# Copyright 2003, 2004, 2005 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# This module defines rules to handle generation of documentation from BoostBook
-# sources.
-#
-# The type of output is controlled by the <format> feature which can have the
-# following values:
-# * html: Generates html documentation. This is the default.
-# * xhtml: Generates xhtml documentation.
-# * htmlhelp: Generates html help output.
-# * onehtml: Generates a single html page.
-# * man: Generates man pages.
-# * pdf: Generates pdf documentation.
-# * ps: Generates postscript output.
-# * docbook: Generates docbook XML.
-# * fo: Generates XSL formating objects.
-# * tests: Extracts test cases from the boostbook XML.
-#
-# <format> is an implicit feature, so for example, typing pdf on the command
-# line is a short-cut for format=pdf.
-
-import build-system ;
-import "class" : new ;
-import common ;
-import feature ;
-import generators ;
-import make ;
-import modules ;
-import os ;
-import path ;
-import print ;
-import project ;
-import property ;
-import property-set ;
-import regex ;
-import scanner ;
-import sequence ;
-import targets ;
-import type ;
-import virtual-target ;
-import xsltproc ;
-
-# Make this module into a project.
-project.initialize $(__name__) ;
-project boostbook ;
-
-.debug-configuration = [ MATCH ^(--debug-configuration)$ : [ modules.peek : ARGV
- ] ] ;
-
-feature.feature format
- : html xhtml htmlhelp onehtml man pdf ps docbook fo tests
- : incidental implicit composite propagated ;
-
-type.register DTDXML : dtdxml ;
-type.register XML : xml ;
-type.register BOOSTBOOK : boostbook : XML ;
-type.register DOCBOOK : docbook : XML ;
-type.register FO : fo : XML ;
-type.register PDF : pdf ;
-type.register PS : ps ;
-type.register XSLT : xsl : XML ;
-type.register HTMLDIR ;
-type.register XHTMLDIR ;
-type.register HTMLHELP ;
-type.register MANPAGES ;
-type.register TESTS : tests ;
-
-
-# Initialize BoostBook support.
-#
-rule init (
- docbook-xsl-dir ? # The DocBook XSL stylesheet directory. If not provided,
- # we use DOCBOOK_XSL_DIR from the environment (if
- # available) or look in standard locations. Otherwise,
- # we let the XML processor load the stylesheets
- # remotely.
-
- : docbook-dtd-dir ? # The DocBook DTD directory. If not provided, we use
- # DOCBOOK_DTD_DIR From the environment (if available) or
- # look in standard locations. Otherwise, we let the XML
- # processor load the DTD remotely.
-
- : boostbook-dir ? # The BoostBook directory with the DTD and XSL subdirs.
-)
-{
- if ! $(.initialized)
- {
- .initialized = true ;
-
- check-boostbook-dir $(boostbook-dir) ;
- find-tools $(docbook-xsl-dir) : $(docbook-dtd-dir) : $(boostbook-dir) ;
-
- # Register generators only if we were called via "using boostbook ;"
- local reg-gen = generators.register-standard ;
- $(reg-gen) boostbook.dtdxml-to-boostbook : DTDXML : XML ;
- $(reg-gen) boostbook.boostbook-to-docbook : XML : DOCBOOK ;
- $(reg-gen) boostbook.boostbook-to-tests : XML : TESTS ;
- $(reg-gen) boostbook.docbook-to-onehtml : DOCBOOK : HTML ;
- $(reg-gen) boostbook.docbook-to-htmldir : DOCBOOK : HTMLDIR ;
- $(reg-gen) boostbook.docbook-to-xhtmldir : DOCBOOK : XHTMLDIR ;
- $(reg-gen) boostbook.docbook-to-htmlhelp : DOCBOOK : HTMLHELP ;
- $(reg-gen) boostbook.docbook-to-manpages : DOCBOOK : MANPAGES ;
- $(reg-gen) boostbook.docbook-to-fo : DOCBOOK : FO ;
-
- # The same about Jamfile main target rules.
- IMPORT $(__name__) : boostbook : : boostbook ;
- }
- else
- {
- if $(docbook-xsl-dir)
- {
- modify-config ;
- .docbook-xsl-dir = [ path.make $(docbook-xsl-dir) ] ;
- check-docbook-xsl-dir ;
- }
- if $(docbook-dtd-dir)
- {
- modify-config ;
- .docbook-dtd-dir = [ path.make $(docbook-dtd-dir) ] ;
- check-docbook-dtd-dir ;
- }
- if $(boostbook-dir)
- {
- modify-config ;
- check-boostbook-dir $(boostbook-dir) ;
- local boostbook-xsl-dir = [ path.glob $(boostbook-dir) : xsl ] ;
- local boostbook-dtd-dir = [ path.glob $(boostbook-dir) : dtd ] ;
- .boostbook-xsl-dir = $(boostbook-xsl-dir[1]) ;
- .boostbook-dtd-dir = $(boostbook-dtd-dir[1]) ;
- check-boostbook-xsl-dir ;
- check-boostbook-dtd-dir ;
- }
- }
-}
-
-
-local rule lock-config ( )
-{
- if ! $(.initialized)
- {
- import errors ;
- errors.user-error BoostBook has not been configured. ;
- }
- if ! $(.config-locked)
- {
- .config-locked = true ;
-
- if $(.error-message)
- {
- print-error $(.error-message) ;
- }
- }
-}
-
-
-local rule modify-config ( )
-{
- if $(.config-locked)
- {
- import errors ;
- errors.user-error BoostBook configuration cannot be changed after it has
- been used. ;
- }
-}
-
-rule print-error ( location message * )
-{
- ECHO error: at $(location) ;
- ECHO error: $(message) ;
- EXIT ;
-}
-
-rule make-error ( message * )
-{
- return [ errors.nearest-user-location ] $(message) ;
-}
-
-
-rule find-boost-in-registry ( keys * )
-{
- local boost-root ;
- for local R in $(keys)
- {
- local installed-boost = [ W32_GETREG
- "HKEY_LOCAL_MACHINE\\SOFTWARE\\$(R)" : "InstallRoot" ] ;
- if $(installed-boost)
- {
- boost-root += [ path.make $(installed-boost) ] ;
- }
- }
- return $(boost-root) ;
-}
-
-
-rule check-docbook-xsl-dir ( )
-{
- if $(.docbook-xsl-dir)
- {
- if ! [ path.glob $(.docbook-xsl-dir) : common/common.xsl ]
- {
- import errors ;
- .error-message = [ make-error BoostBook: could not find docbook XSL stylesheets
- in: [ path.native $(.docbook-xsl-dir) ] ] ;
- }
- else if $(.debug-configuration)
- {
- ECHO notice: BoostBook: found docbook XSL stylesheets in: [
- path.native $(.docbook-xsl-dir) ] ;
- }
- }
-}
-
-
-rule check-docbook-dtd-dir ( )
-{
- if $(.docbook-dtd-dir)
- {
- if ! [ path.glob $(.docbook-dtd-dir) : docbookx.dtd ]
- {
- import errors ;
- .error-message = [ make-error BoostBook: could not find docbook DTD in: [
- path.native $(.docbook-dtd-dir) ] ] ;
- }
- else if $(.debug-configuration)
- {
- ECHO notice: BoostBook: found docbook DTD in: [ path.native
- $(.docbook-dtd-dir) ] ;
- }
- }
-}
-
-
-rule check-boostbook-xsl-dir ( )
-{
- if ! $(.boostbook-xsl-dir)
- {
- .error-message = [ make-error BoostBook: could not find boostbook XSL stylesheets. ] ;
- }
- else if ! [ path.glob $(.boostbook-xsl-dir) : docbook.xsl ]
- {
- .error-message = [ make-error BoostBook: could not find docbook XSL stylesheets in:
- [ path.native $(.boostbook-xsl-dir) ] ] ;
- }
- else if $(.debug-configuration)
- {
- ECHO notice: BoostBook: found boostbook XSL stylesheets in: [
- path.native $(.boostbook-xsl-dir) ] ;
- }
-}
-
-
-rule check-boostbook-dtd-dir ( )
-{
- if ! $(.boostbook-dtd-dir)
- {
- .error-message = [ make-error BoostBook: could not find boostbook DTD. ] ;
- }
- else if ! [ path.glob $(.boostbook-dtd-dir) : boostbook.dtd ]
- {
- .error-message = [ make-error BoostBook: could not find boostbook DTD in: [
- path.native $(.boostbook-dtd-dir) ] ] ;
- }
- else if $(.debug-configuration)
- {
- ECHO notice: BoostBook: found boostbook DTD in: [ path.native
- $(.boostbook-dtd-dir) ] ;
- }
-}
-
-
-rule check-boostbook-dir ( boostbook-dir ? )
-{
- if $(boostbook-dir) && ! [ path.glob $(boostbook-dir) : xsl ]
- {
- import errors ;
- .error-message = [ make-error BoostBook: could not find boostbook in: [ path.native
- $(boostbook-dir) ] ] ;
- }
-}
-
-
-rule find-tools ( docbook-xsl-dir ? : docbook-dtd-dir ? : boostbook-dir ? )
-{
- docbook-xsl-dir ?= [ modules.peek : DOCBOOK_XSL_DIR ] ;
- docbook-dtd-dir ?= [ modules.peek : DOCBOOK_DTD_DIR ] ;
- boostbook-dir ?= [ modules.peek : BOOSTBOOK_DIR ] ;
-
- # Look for the boostbook stylesheets relative to BOOST_ROOT and Boost.Build.
- local boost-build-root = [ path.make [ build-system.location ] ] ;
- local boostbook-search-dirs = [ path.join $(boost-build-root) .. .. ] ;
-
- local boost-root = [ modules.peek : BOOST_ROOT ] ;
- if $(boost-root)
- {
- boostbook-search-dirs += [ path.join [ path.make $(boost-root) ] tools ]
- ;
- }
- boostbook-dir ?= [ path.glob $(boostbook-search-dirs) : boostbook* ] ;
-
- # Try to find the tools in platform specific locations.
- if [ os.name ] = NT
- {
- # If installed by the Boost installer.
- local boost-root = ;
-
- local boost-installer-versions = snapshot cvs 1.33.0 ;
- local boost-consulting-installer-versions = 1.33.1 1.34.0 1.34.1 ;
- local boostpro-installer-versions =
- 1.35.0 1.36.0 1.37.0 1.38.0 1.39.0 1.40.0 1.41.0 1.42.0
- 1.43.0 1.44.0 1.45.0 1.46.0 1.47.0 1.48.0 1.49.0 1.50.0 ;
-
- local old-installer-root = [ find-boost-in-registry
- Boost.org\\$(boost-installer-versions) ] ;
-
- # Make sure that the most recent version is searched for first.
- boost-root += [ sequence.reverse [ find-boost-in-registry
- Boost-Consulting.com\\$(boost-consulting-installer-versions)
- boostpro.com\\$(boostpro-installer-versions) ] ] ;
-
- # Plausible locations.
- local root = [ PWD ] ;
- while $(root) != $(root:D) { root = $(root:D) ; }
- root = [ path.make $(root) ] ;
- local search-dirs ;
- local docbook-search-dirs ;
- for local p in $(boost-root)
- {
- search-dirs += [ path.join $(p) tools ] ;
- }
- for local p in $(old-installer-root)
- {
- search-dirs += [ path.join $(p) share ] ;
- docbook-search-dirs += [ path.join $(p) share ] ;
- }
- search-dirs += [ path.join $(root) Boost tools ] ;
- search-dirs += [ path.join $(root) Boost share ] ;
- docbook-search-dirs += [ path.join $(root) Boost share ] ;
-
- docbook-xsl-dir ?= [ path.glob $(docbook-search-dirs) : docbook-xsl* ] ;
- docbook-dtd-dir ?= [ path.glob $(docbook-search-dirs) : docbook-xml* ] ;
- boostbook-dir ?= [ path.glob $(search-dirs) : boostbook* ] ;
- }
- else
- {
- # Plausible locations.
-
- local share = /usr/local/share /usr/share /opt/share /opt/local/share ;
- local dtd-versions = 4.2 ;
-
- docbook-xsl-dir ?= [ path.glob $(share) : docbook-xsl* ] ;
- docbook-xsl-dir ?= [ path.glob $(share)/sgml/docbook : xsl-stylesheets ]
- ;
- docbook-xsl-dir ?= [ path.glob $(share)/xsl : docbook* ] ;
-
- docbook-dtd-dir ?= [ path.glob $(share) : docbook-xml* ] ;
- docbook-dtd-dir ?= [ path.glob $(share)/sgml/docbook :
- xml-dtd-$(dtd-versions)* ] ;
- docbook-dtd-dir ?= [ path.glob $(share)/xml/docbook : $(dtd-versions) ]
- ;
-
- boostbook-dir ?= [ path.glob $(share) : boostbook* ] ;
-
- # Ubuntu Linux.
- docbook-xsl-dir ?= [ path.glob /usr/share/xml/docbook/stylesheet :
- nwalsh ] ;
- docbook-dtd-dir ?= [ path.glob /usr/share/xml/docbook/schema/dtd :
- $(dtd-versions) ] ;
-
- # SUSE.
- docbook-xsl-dir ?= [ path.glob /usr/share/xml/docbook/stylesheet/nwalsh
- : current ] ;
- }
-
- if $(docbook-xsl-dir)
- {
- .docbook-xsl-dir = [ path.make $(docbook-xsl-dir[1]) ] ;
- }
- if $(docbook-dtd-dir)
- {
- .docbook-dtd-dir = [ path.make $(docbook-dtd-dir[1]) ] ;
- }
-
- if $(.debug-configuration)
- {
- ECHO notice: Boost.Book: searching XSL/DTD "in" ;
- ECHO notice: [ sequence.transform path.native : $(boostbook-dir) ] ;
- }
- local boostbook-xsl-dir ;
- for local dir in $(boostbook-dir)
- {
- boostbook-xsl-dir += [ path.glob $(dir) : xsl ] ;
- }
- local boostbook-dtd-dir ;
- for local dir in $(boostbook-dir)
- {
- boostbook-dtd-dir += [ path.glob $(dir) : dtd ] ;
- }
- .boostbook-xsl-dir = $(boostbook-xsl-dir[1]) ;
- .boostbook-dtd-dir = $(boostbook-dtd-dir[1]) ;
-
- check-docbook-xsl-dir ;
- check-docbook-dtd-dir ;
- check-boostbook-xsl-dir ;
- check-boostbook-dtd-dir ;
-}
-
-
-rule xsl-dir
-{
- lock-config ;
- return $(.boostbook-xsl-dir) ;
-}
-
-
-rule dtd-dir
-{
- lock-config ;
- return $(.boostbook-dtd-dir) ;
-}
-
-
-rule docbook-xsl-dir
-{
- lock-config ;
- return $(.docbook-xsl-dir) ;
-}
-
-
-rule docbook-dtd-dir
-{
- lock-config ;
- return $(.docbook-dtd-dir) ;
-}
-
-
-rule dtdxml-to-boostbook ( target : source : properties * )
-{
- lock-config ;
- xsltproc.xslt $(target) : $(source)
- "$(.boostbook-xsl-dir)/dtd/dtd2boostbook.xsl" : $(properties) ;
-}
-
-
-rule boostbook-to-docbook ( target : source : properties * )
-{
- lock-config ;
- local stylesheet = [ path.native $(.boostbook-xsl-dir)/docbook.xsl ] ;
- xsltproc.xslt $(target) : $(source) $(stylesheet) : $(properties) ;
-}
-
-
-rule docbook-to-onehtml ( target : source : properties * )
-{
- lock-config ;
- local stylesheet = [ path.native $(.boostbook-xsl-dir)/html-single.xsl ] ;
- xsltproc.xslt $(target) : $(source) $(stylesheet) : $(properties) ;
-}
-
-
-rule docbook-to-htmldir ( target : source : properties * )
-{
- lock-config ;
- local stylesheet = [ path.native $(.boostbook-xsl-dir)/html.xsl ] ;
- xsltproc.xslt-dir $(target) : $(source) $(stylesheet) : $(properties) : html
- ;
-}
-
-
-rule docbook-to-xhtmldir ( target : source : properties * )
-{
- lock-config ;
- local stylesheet = [ path.native $(.boostbook-xsl-dir)/xhtml.xsl ] ;
- xsltproc.xslt-dir $(target) : $(source) $(stylesheet) : $(properties) :
- xhtml ;
-}
-
-
-rule docbook-to-htmlhelp ( target : source : properties * )
-{
- lock-config ;
- local stylesheet = [ path.native $(.boostbook-xsl-dir)/html-help.xsl ] ;
- xsltproc.xslt-dir $(target) : $(source) $(stylesheet) : $(properties) :
- htmlhelp ;
-}
-
-
-rule docbook-to-manpages ( target : source : properties * )
-{
- lock-config ;
- local stylesheet = [ path.native $(.boostbook-xsl-dir)/manpages.xsl ] ;
- xsltproc.xslt-dir $(target) : $(source) $(stylesheet) : $(properties) : man
- ;
-}
-
-
-rule docbook-to-fo ( target : source : properties * )
-{
- lock-config ;
- local stylesheet = [ path.native $(.boostbook-xsl-dir)/fo.xsl ] ;
- xsltproc.xslt $(target) : $(source) $(stylesheet) : $(properties) ;
-}
-
-
-rule format-catalog-path ( path )
-{
- local result = $(path) ;
- if [ xsltproc.is-cygwin ]
- {
- if [ os.name ] = NT
- {
- drive = [ MATCH ^/(.):(.*)$ : $(path) ] ;
- result = /cygdrive/$(drive[1])$(drive[2]) ;
- }
- }
- else
- {
- if [ os.name ] = CYGWIN
- {
- local native-path = [ path.native $(path) ] ;
- result = [ path.make $(native-path:W) ] ;
- }
- }
- return [ regex.replace $(result) " " "%20" ] ;
-}
-
-
-rule generate-xml-catalog ( target : sources * : properties * )
-{
- print.output $(target) ;
-
- # BoostBook DTD catalog entry.
- local boostbook-dtd-dir = [ boostbook.dtd-dir ] ;
- if $(boostbook-dtd-dir)
- {
- boostbook-dtd-dir = [ format-catalog-path $(boostbook-dtd-dir) ] ;
- }
-
- print.text
- "<?xml version=\"1.0\"?>"
- "<!DOCTYPE catalog "
- " PUBLIC \"-//OASIS/DTD Entity Resolution XML Catalog V1.0//EN\""
- " \"http://www.oasis-open.org/committees/entity/release/1.0/catalog.dtd\">"
- "<catalog xmlns=\"urn:oasis:names:tc:entity:xmlns:xml:catalog\">"
- " <rewriteURI uriStartString=\"http://www.boost.org/tools/boostbook/dtd/\" rewritePrefix=\"file://$(boostbook-dtd-dir)/\"/>"
- : true ;
-
- local docbook-xsl-dir = [ boostbook.docbook-xsl-dir ] ;
- if ! $(docbook-xsl-dir)
- {
- ECHO "BoostBook warning: no DocBook XSL directory specified." ;
- ECHO " If you have the DocBook XSL stylesheets installed, please " ;
- ECHO " set DOCBOOK_XSL_DIR to the stylesheet directory on either " ;
- ECHO " the command line (via -sDOCBOOK_XSL_DIR=...) or in a " ;
- ECHO " Boost.Jam configuration file. The DocBook XSL stylesheets " ;
- ECHO " are available here: http://docbook.sourceforge.net/ " ;
- ECHO " Stylesheets will be downloaded on-the-fly (very slow!) " ;
- }
- else
- {
- docbook-xsl-dir = [ format-catalog-path $(docbook-xsl-dir) ] ;
- print.text " <rewriteURI uriStartString=\"http://docbook.sourceforge.net/release/xsl/current/\" rewritePrefix=\"file://$(docbook-xsl-dir)/\"/>" ;
- }
-
- local docbook-dtd-dir = [ boostbook.docbook-dtd-dir ] ;
- if ! $(docbook-dtd-dir)
- {
- ECHO "BoostBook warning: no DocBook DTD directory specified." ;
- ECHO " If you have the DocBook DTD installed, please set " ;
- ECHO " DOCBOOK_DTD_DIR to the DTD directory on either " ;
- ECHO " the command line (via -sDOCBOOK_DTD_DIR=...) or in a " ;
- ECHO " Boost.Jam configuration file. The DocBook DTD is available " ;
- ECHO " here: http://www.oasis-open.org/docbook/xml/4.2/index.shtml" ;
- ECHO " The DTD will be downloaded on-the-fly (very slow!) " ;
- }
- else
- {
- docbook-dtd-dir = [ format-catalog-path $(docbook-dtd-dir) ] ;
- print.text " <rewriteURI uriStartString=\"http://www.oasis-open.org/docbook/xml/4.2/\" rewritePrefix=\"file://$(docbook-dtd-dir)/\"/>" ;
- }
-
- print.text "</catalog>" ;
-}
-
-
-# Returns information about the global XML catalog target, creating it lazily if
-# needed. To get the global catalog generated only once we do not create it in
-# every project that requests it but instead only create it based on the first
-# project requesting it and then reuse it from there for any later requests.
-#
-# To get 'as close as possible' to having the global catalog stored in the same
-# location independent of which folder our build was run from, we assign its
-# target to the given project's base Jamroot project. This works correctly as
-# long as we know the passed project is not standalone or one of Boost Build's
-# configuration module projects, as those to not have a Jamroot project in their
-# parent chain. Note also that we can still get our targets generated in
-# different folders in case when one build project references a target from
-# another build project with its own separate Jamroot.
-#
-# FIXME: Ideally the catalog target should be created as part of the boostbook
-# project and stored in some central location for all used standalone pojects,
-# shared between all builds made on that system. This however would require much
-# more though to add the necessary changes to Boost Build's internal design.
-#
-local rule xml-catalog ( project )
-{
- if ! $(.xml-catalog)
- {
- local project-module = [ $(project).project-module ] ;
- local root-module = [ project.get-jamroot-module $(project-module) ] ;
- if ! $(root-module)
- {
- import errors ;
- if [ project.is-config-module $(project-module) ]
- {
- errors.user-error boostbook targets can not be declared in Boost
- Build's configuration modules. ;
- }
- else
- {
- errors.user-error boostbook targets can not be declared in
- standalone projects. : use a Jamfile/Jamroot project
- instead. ;
- }
- }
- local root-project = [ project.target $(root-module) ] ;
-
- .xml-catalog = [ virtual-target.register [ new file-target
- boostbook_catalog : XML : $(root-project) : [ new action :
- boostbook.generate-xml-catalog ] ] ] ;
- .xml-catalog-file = [ $(.xml-catalog).path ] [ $(.xml-catalog).name ] ;
- .xml-catalog-file = $(.xml-catalog-file:J=/) ;
- }
- return $(.xml-catalog) $(.xml-catalog-file) ;
-}
-
-
-class boostbook-target-class : basic-target
-{
- import generators ;
- import property-set ;
- import virtual-target ;
-
- rule construct ( name : sources * : property-set )
- {
- # Generate the catalog, but only once.
- IMPORT boostbook : xml-catalog : $(__name__) : boostbook.xml-catalog ;
- local global-catalog = [ boostbook.xml-catalog [ project ] ] ;
- local catalog = $(global-catalog[1]) ;
- local catalog-file = $(global-catalog[2]) ;
- local targets ;
-
- # Add the catalog to the property set.
- property-set = [ $(property-set).add-raw <catalog>$(catalog-file) ] ;
-
- local type = none ;
- local manifest ;
- local format = [ $(property-set).get <format> ] ;
- switch $(format)
- {
- case html : type = HTMLDIR ; manifest = HTML.manifest ;
- case xhtml : type = XHTMLDIR ; manifest = HTML.manifest ;
- case htmlhelp : type = HTMLHELP ; manifest = HTML.manifest ;
- case onehtml : type = HTML ;
- case man : type = MANPAGES ; manifest = man.manifest ;
- case docbook : type = DOCBOOK ;
- case fo : type = FO ;
- case pdf : type = PDF ;
- case ps : type = PS ;
- case tests : type = TESTS ;
- }
-
- local target ;
- if $(manifest)
- {
- # Sources --> DOCBOOK.
- local docbook-target = [ generators.construct [ project ] : DOCBOOK
- : $(property-set) : $(sources) ] ;
- docbook-target = $(docbook-target[2]) ;
- $(docbook-target).depends $(catalog) ;
-
- # DOCBOOK --> type.
- target = [ generators.construct [ project ] $(name)_$(manifest) :
- $(type) : [ $(property-set).add-raw
- <xsl:param>manifest=$(name)_$(manifest) ] : $(docbook-target) ]
- ;
- target = $(target[2]) ;
- local name = [ $(property-set).get <name> ] ;
- name ?= $(format) ;
- $(target).set-path $(name) ;
- }
- else
- {
- # Sources --> type.
- target = [ generators.construct [ project ] : $(type) :
- $(property-set) : $(sources) ] ;
- target = $(target[2]) ;
- if ! $(target)
- {
- import errors ;
- errors.error Cannot build documentation type '$(format)'. ;
- }
- }
- $(target).depends $(catalog) ;
-
- return [ property-set.empty ] $(target) ;
- }
-}
-
-
-# Declare a boostbook target.
-#
-rule boostbook ( target-name : sources * : requirements * : default-build * )
-{
- return [ targets.create-metatarget boostbook-target-class :
- [ project.current ] : $(target-name) : $(sources) : $(requirements) :
- $(default-build) ] ;
-}
-
-
-rule boostbook-to-tests ( target : source : properties * )
-{
- lock-config ;
- local boost_root = [ modules.peek : BOOST_ROOT ] ;
- local native-path = [ path.native [ path.join $(.boostbook-xsl-dir) testing
- Jamfile ] ] ;
- local stylesheet = $(native-path:S=.xsl) ;
- xsltproc.xslt $(target) : $(source) $(stylesheet) : $(properties)
- <xsl:param>boost.root=$(boost_root) ;
-}
-
-
-#############################################################################
-# Dependency scanners
-#############################################################################
-# XInclude scanner. Mostly stolen from c-scanner. :)
-# Note that this assumes an "xi" prefix for XIncludes. This is not always the
-# case for XML documents, but we assume it is true for anything we encounter.
-#
-class xinclude-scanner : scanner
-{
- import scanner ;
-
- rule __init__ ( includes * )
- {
- scanner.__init__ ;
- self.includes = $(includes) ;
- }
-
- rule pattern ( )
- {
- return "xi:include[ ]*href=\"([^\"]*)\"" ;
- }
-
- rule process ( target : matches * : binding )
- {
- local target_path = [ NORMALIZE_PATH $(binding:D) ] ;
-
- NOCARE $(matches) ;
- INCLUDES $(target) : $(matches) ;
- SEARCH on $(matches) = $(target_path) $(self.includes:G=) ;
-
- scanner.propagate $(__name__) : $(matches) : $(target) ;
- }
-}
-
-scanner.register xinclude-scanner : xsl:path ;
-type.set-scanner XML : xinclude-scanner ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/borland.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/borland.jam b/ext/kenlm/jam-files/boost-build/tools/borland.jam
deleted file mode 100644
index 6e43ca9..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/borland.jam
+++ /dev/null
@@ -1,220 +0,0 @@
-# Copyright 2005 Dave Abrahams
-# Copyright 2003 Rene Rivera
-# Copyright 2003, 2004, 2005 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-# Support for the Borland's command line compiler
-
-import property ;
-import generators ;
-import os ;
-import toolset : flags ;
-import feature : get-values ;
-import type ;
-import common ;
-
-feature.extend toolset : borland ;
-
-rule init ( version ? : command * : options * )
-{
- local condition = [ common.check-init-parameters borland :
- version $(version) ] ;
-
- local command = [ common.get-invocation-command borland : bcc32.exe
- : $(command) ] ;
-
- common.handle-options borland : $(condition) : $(command) : $(options) ;
-
- if $(command)
- {
- command = [ common.get-absolute-tool-path $(command[-1]) ] ;
- }
- root = $(command:D) ;
-
- flags borland.compile STDHDRS $(condition) : $(root)/include/ ;
- flags borland.link STDLIBPATH $(condition) : $(root)/lib ;
- flags borland.link RUN_PATH $(condition) : $(root)/bin ;
- flags borland .root $(condition) : $(root)/bin/ ;
-}
-
-
-# A borland-specific target type
-type.register BORLAND.TDS : tds ;
-
-# Declare generators
-
-generators.register-linker borland.link : OBJ SEARCHED_LIB STATIC_LIB IMPORT_LIB : EXE : <toolset>borland ;
-generators.register-linker borland.link.dll : OBJ SEARCHED_LIB STATIC_LIB IMPORT_LIB : SHARED_LIB IMPORT_LIB : <toolset>borland ;
-
-generators.register-archiver borland.archive : OBJ : STATIC_LIB : <toolset>borland ;
-generators.register-c-compiler borland.compile.c++ : CPP : OBJ : <toolset>borland ;
-generators.register-c-compiler borland.compile.c : C : OBJ : <toolset>borland ;
-generators.register-standard borland.asm : ASM : OBJ : <toolset>borland ;
-
-# Declare flags
-
-flags borland.compile OPTIONS <debug-symbols>on : -v ;
-flags borland.link OPTIONS <debug-symbols>on : -v ;
-
-flags borland.compile OPTIONS <optimization>off : -Od ;
-flags borland.compile OPTIONS <optimization>speed : -O2 ;
-flags borland.compile OPTIONS <optimization>space : -O1 ;
-
-if $(.BORLAND_HAS_FIXED_INLINING_BUGS)
-{
- flags borland CFLAGS <inlining>off : -vi- ;
- flags borland CFLAGS <inlining>on : -vi -w-inl ;
- flags borland CFLAGS <inlining>full : -vi -w-inl ;
-}
-else
-{
- flags borland CFLAGS : -vi- ;
-}
-
-flags borland.compile OPTIONS <warnings>off : -w- ;
-flags borland.compile OPTIONS <warnings>all : -w ;
-flags borland.compile OPTIONS <warnings-as-errors>on : -w! ;
-
-
-# Deal with various runtime configs...
-
-# This should be not for DLL
-flags borland OPTIONS <user-interface>console : -tWC ;
-
-# -tWR sets -tW as well, so we turn it off here and then turn it
-# on again later if we need it:
-flags borland OPTIONS <runtime-link>shared : -tWR -tWC ;
-flags borland OPTIONS <user-interface>gui : -tW ;
-
-flags borland OPTIONS <main-target-type>LIB/<link>shared : -tWD ;
-# Hmm.. not sure what's going on here.
-flags borland OPTIONS : -WM- ;
-flags borland OPTIONS <threading>multi : -tWM ;
-
-
-
-flags borland.compile OPTIONS <cxxflags> ;
-flags borland.compile DEFINES <define> ;
-flags borland.compile INCLUDES <include> ;
-
-flags borland NEED_IMPLIB <main-target-type>LIB/<link>shared : "" ;
-
-#
-# for C++ compiles the following options are turned on by default:
-#
-# -j5 stops after 5 errors
-# -g255 allow an unlimited number of warnings
-# -q no banner
-# -c compile to object
-# -P C++ code regardless of file extention
-# -a8 8 byte alignment, this option is on in the IDE by default
-# and effects binary compatibility.
-#
-
-# -U$(UNDEFS) -D$(DEFINES) $(CFLAGS) $(C++FLAGS) -I"$(HDRS)" -I"$(STDHDRS)" -o"$(<)" "$(>)"
-
-
-actions compile.c++
-{
- "$(CONFIG_COMMAND)" -j5 -g255 -q -c -P -a8 -Vx- -Ve- -b- $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -I"$(STDHDRS)" -o"$(<)" "$(>)"
-}
-
-# For C, we don't pass -P flag
-actions compile.c
-{
- "$(CONFIG_COMMAND)" -j5 -g255 -q -c -a8 -Vx- -Ve- -b- $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -I"$(STDHDRS)" -o"$(<)" "$(>)"
-}
-
-
-# Declare flags and action for linking
-toolset.flags borland.link OPTIONS <debug-symbols>on : -v ;
-toolset.flags borland.link LIBRARY_PATH <library-path> ;
-toolset.flags borland.link FINDLIBS_ST <find-static-library> ;
-toolset.flags borland.link FINDLIBS_SA <find-shared-library> ;
-toolset.flags borland.link LIBRARIES <library-file> ;
-
-flags borland.link OPTIONS <linkflags> ;
-flags borland.link OPTIONS <link>shared : -tWD ;
-
-flags borland.link LIBRARY_PATH_OPTION <toolset>borland : -L : unchecked ;
-flags borland.link LIBRARY_OPTION <toolset>borland : "" : unchecked ;
-
-
-
-# bcc32 needs to have ilink32 in the path in order to invoke it, so explicitly
-# specifying $(BCC_TOOL_PATH)bcc32 doesn't help. You need to add
-# $(BCC_TOOL_PATH) to the path
-# The NEED_IMPLIB variable controls whether we need to invoke implib.
-
-flags borland.archive AROPTIONS <archiveflags> ;
-
-# Declare action for archives. We don't use response file
-# since it's hard to get "+-" there.
-# The /P256 increases 'page' size -- with too low
-# values tlib fails when building large applications.
-# CONSIDER: don't know what 'together' is for...
-actions updated together piecemeal archive
-{
- $(.set-path)$(.root:W)$(.old-path)
- tlib $(AROPTIONS) /P256 /u /a /C "$(<:W)" +-"$(>:W)"
-}
-
-
-if [ os.name ] = CYGWIN
-{
- .set-path = "cmd /S /C set \"PATH=" ;
- .old-path = ";%PATH%\" \"&&\"" ;
-
-
- # Couldn't get TLIB to stop being confused about pathnames
- # containing dashes (it seemed to treat them as option separators
- # when passed through from bash), so we explicitly write the
- # command into a .bat file and execute that. TLIB is also finicky
- # about pathname style! Forward slashes, too, are treated as
- # options.
- actions updated together piecemeal archive
- {
- chdir $(<:D)
- echo +-$(>:BS) > $(<:BS).rsp
- $(.set-path)$(.root)$(.old-path) "tlib.exe" $(AROPTIONS) /P256 /C $(<:BS) @$(<:BS).rsp && $(RM) $(<:BS).rsp
- }
-}
-else if [ os.name ] = NT
-{
- .set-path = "set \"PATH=" ;
- .old-path = ";%PATH%\"
- " ;
-}
-else
-{
- .set-path = "PATH=\"" ;
- .old-path = "\":$PATH
- export PATH
- " ;
-}
-
-RM = [ common.rm-command ] ;
-
-nl = "
-" ;
-
-actions link
-{
- $(.set-path)$(.root:W)$(.old-path) "$(CONFIG_COMMAND)" -v -q $(OPTIONS) -L"$(LIBRARY_PATH:W)" -L"$(STDLIBPATH:W)" -e"$(<[1]:W)" @"@($(<[1]:W).rsp:E=$(nl)"$(>)" $(nl)$(LIBRARIES) $(nl)"$(LIBRARY_OPTION)$(FINDLIBS_ST:S=.lib)" $(nl)"$(LIBRARY_OPTION)$(FINDLIBS_SA:S=.lib)")"
-}
-
-
-actions link.dll bind LIBRARIES RSP
-{
- $(.set-path)$(.root:W)$(.old-path) "$(CONFIG_COMMAND)" -v -q $(OPTIONS) -L"$(LIBRARY_PATH:W)" -L"$(STDLIBPATH:W)" -e"$(<[1]:W)" @"@($(<[1]:W).rsp:E=$(nl)"$(>)" $(nl)$(LIBRARIES) $(nl)"$(LIBRARY_OPTION)$(FINDLIBS_ST:S=.lib)" $(nl)"$(LIBRARY_OPTION)$(FINDLIBS_SA:S=.lib)")" && "$(.root)implib" "$(<[2]:W)" "$(<[1]:W)"
-}
-
-# It seems impossible to specify output file with directory when compiling
-# asm files using bcc32, so use tasm32 directly.
-# /ml makes all symbol names case-sensitive
-actions asm
-{
- $(.set-path)$(.root:W)$(.old-path) tasm32.exe /ml "$(>)" "$(<)"
-}
-
[43/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/build/type.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/build/type.jam b/ext/kenlm/jam-files/boost-build/build/type.jam
deleted file mode 100644
index e8cc44e..0000000
--- a/ext/kenlm/jam-files/boost-build/build/type.jam
+++ /dev/null
@@ -1,401 +0,0 @@
-# Copyright 2002, 2003 Dave Abrahams
-# Copyright 2002, 2003, 2004, 2005, 2006 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-# Deals with target type declaration and defines target class which supports
-# typed targets.
-
-import "class" : new ;
-import feature ;
-import generators : * ;
-import os ;
-import project ;
-import property ;
-import scanner ;
-
-# The following import would create a circular dependency:
-# project -> project-root -> builtin -> type -> targets -> project
-# import targets ;
-
-# The feature is optional so it would never get added implicitly. It is used
-# only for internal purposes and in all cases we want to use it explicitly.
-feature.feature target-type : : composite optional ;
-
-feature.feature main-target-type : : optional incidental ;
-feature.feature base-target-type : : composite optional free ;
-
-
-# Registers a target type, possible derived from a 'base-type'. Providing a list
-# of 'suffixes' here is a shortcut for separately calling the register-suffixes
-# rule with the given suffixes and the set-generated-target-suffix rule with the
-# first given suffix.
-#
-rule register ( type : suffixes * : base-type ? )
-{
- # Type names cannot contain hyphens, because when used as feature-values
- # they would be interpreted as composite features which need to be
- # decomposed.
- switch $(type)
- {
- case *-* :
- import errors ;
- errors.error "type name \"$(type)\" contains a hyphen" ;
- }
-
- if $(type) in $(.types)
- {
- import errors ;
- errors.error "Type $(type) is already registered." ;
- }
-
- {
- .types += $(type) ;
- .base.$(type) = $(base-type) ;
- .derived.$(base-type) += $(type) ;
- .bases.$(type) = $(type) $(.bases.$(base-type)) ;
-
- # Store suffixes for generated targets.
- .suffixes.$(type) = [ new property-map ] ;
-
- # Store prefixes for generated targets (e.g. "lib" for library).
- .prefixes.$(type) = [ new property-map ] ;
-
- if $(suffixes)-is-defined
- {
- # Specify mapping from suffixes to type.
- register-suffixes $(suffixes) : $(type) ;
- # By default generated targets of 'type' will use the first of
- #'suffixes'. This may be overriden.
- set-generated-target-suffix $(type) : : $(suffixes[1]) ;
- }
-
- feature.extend target-type : $(type) ;
- feature.extend main-target-type : $(type) ;
- feature.extend base-target-type : $(type) ;
-
- feature.compose <target-type>$(type) : $(base-type:G=<base-target-type>) ;
- feature.compose <base-target-type>$(type) : <base-target-type>$(base-type) ;
-
- # We used to declare the main target rule only when a 'main' parameter
- # has been specified. However, it is hard to decide that a type will
- # *never* need a main target rule and so from time to time we needed to
- # make yet another type 'main'. So now a main target rule is defined for
- # each type.
- main-rule-name = [ type-to-rule-name $(type) ] ;
- .main-target-type.$(main-rule-name) = $(type) ;
- IMPORT $(__name__) : main-target-rule : : $(main-rule-name) ;
-
- # Adding a new derived type affects generator selection so we need to
- # make the generator selection module update any of its cached
- # information related to a new derived type being defined.
- generators.update-cached-information-with-a-new-type $(type) ;
- }
-}
-
-
-# Given a type, returns the name of the main target rule which creates targets
-# of that type.
-#
-rule type-to-rule-name ( type )
-{
- # Lowercase everything. Convert underscores to dashes.
- import regex ;
- local n = [ regex.split $(type:L) "_" ] ;
- return $(n:J=-) ;
-}
-
-
-# Given a main target rule name, returns the type for which it creates targets.
-#
-rule type-from-rule-name ( rule-name )
-{
- return $(.main-target-type.$(rule-name)) ;
-}
-
-
-# Specifies that files with suffix from 'suffixes' be recognized as targets of
-# type 'type'. Issues an error if a different type is already specified for any
-# of the suffixes.
-#
-rule register-suffixes ( suffixes + : type )
-{
- for local s in $(suffixes)
- {
- if ! $(.type.$(s))
- {
- .type.$(s) = $(type) ;
- }
- else if $(.type.$(s)) != $(type)
- {
- import errors ;
- errors.error Attempting to specify multiple types for suffix
- \"$(s)\" : "Old type $(.type.$(s)), New type $(type)" ;
- }
- }
-}
-
-
-# Returns true iff type has been registered.
-#
-rule registered ( type )
-{
- if $(type) in $(.types)
- {
- return true ;
- }
-}
-
-
-# Issues an error if 'type' is unknown.
-#
-rule validate ( type )
-{
- if ! [ registered $(type) ]
- {
- import errors ;
- errors.error "Unknown target type $(type)" ;
- }
-}
-
-
-# Sets a scanner class that will be used for this 'type'.
-#
-rule set-scanner ( type : scanner )
-{
- validate $(type) ;
- .scanner.$(type) = $(scanner) ;
-}
-
-
-# Returns a scanner instance appropriate to 'type' and 'properties'.
-#
-rule get-scanner ( type : property-set )
-{
- if $(.scanner.$(type))
- {
- return [ scanner.get $(.scanner.$(type)) : $(property-set) ] ;
- }
-}
-
-
-# Returns a base type for the given type or nothing in case the given type is
-# not derived.
-#
-rule base ( type )
-{
- return $(.base.$(type)) ;
-}
-
-
-# Returns the given type and all of its base types in order of their distance
-# from type.
-#
-rule all-bases ( type )
-{
- return $(.bases.$(type)) ;
-}
-
-
-# Returns the given type and all of its derived types in order of their distance
-# from type.
-#
-rule all-derived ( type )
-{
- local result = $(type) ;
- for local d in $(.derived.$(type))
- {
- result += [ all-derived $(d) ] ;
- }
- return $(result) ;
-}
-
-
-# Returns true if 'type' is equal to 'base' or has 'base' as its direct or
-# indirect base.
-#
-rule is-derived ( type base )
-{
- if $(base) in $(.bases.$(type))
- {
- return true ;
- }
-}
-
-# Returns true if 'type' is either derived from or is equal to 'base'.
-#
-# TODO: It might be that is-derived and is-subtype were meant to be different
-# rules - one returning true for type = base and one not, but as currently
-# implemented they are actually the same. Clean this up.
-#
-rule is-subtype ( type base )
-{
- return [ is-derived $(type) $(base) ] ;
-}
-
-
-
-
-# Sets a file suffix to be used when generating a target of 'type' with the
-# specified properties. Can be called with no properties if no suffix has
-# already been specified for the 'type'. The 'suffix' parameter can be an empty
-# string ("") to indicate that no suffix should be used.
-#
-# Note that this does not cause files with 'suffix' to be automatically
-# recognized as being of 'type'. Two different types can use the same suffix for
-# their generated files but only one type can be auto-detected for a file with
-# that suffix. User should explicitly specify which one using the
-# register-suffixes rule.
-#
-rule set-generated-target-suffix ( type : properties * : suffix )
-{
- set-generated-target-ps suffix : $(type) : $(properties) : $(suffix) ;
-}
-
-
-# Change the suffix previously registered for this type/properties combination.
-# If suffix is not yet specified, sets it.
-#
-rule change-generated-target-suffix ( type : properties * : suffix )
-{
- change-generated-target-ps suffix : $(type) : $(properties) : $(suffix) ;
-}
-
-
-# Returns the suffix used when generating a file of 'type' with the given
-# properties.
-#
-rule generated-target-suffix ( type : property-set )
-{
- return [ generated-target-ps suffix : $(type) : $(property-set) ] ;
-}
-
-
-# Sets a target prefix that should be used when generating targets of 'type'
-# with the specified properties. Can be called with empty properties if no
-# prefix for 'type' has been specified yet.
-#
-# The 'prefix' parameter can be empty string ("") to indicate that no prefix
-# should be used.
-#
-# Usage example: library names use the "lib" prefix on unix.
-#
-rule set-generated-target-prefix ( type : properties * : prefix )
-{
- set-generated-target-ps prefix : $(type) : $(properties) : $(prefix) ;
-}
-
-
-# Change the prefix previously registered for this type/properties combination.
-# If prefix is not yet specified, sets it.
-#
-rule change-generated-target-prefix ( type : properties * : prefix )
-{
- change-generated-target-ps prefix : $(type) : $(properties) : $(prefix) ;
-}
-
-
-rule generated-target-prefix ( type : property-set )
-{
- return [ generated-target-ps prefix : $(type) : $(property-set) ] ;
-}
-
-
-# Common rules for prefix/suffix provisioning follow.
-
-local rule set-generated-target-ps ( ps : type : properties * : psval )
-{
- $(.$(ps)es.$(type)).insert $(properties) : $(psval) ;
-}
-
-
-local rule change-generated-target-ps ( ps : type : properties * : psval )
-{
- local prev = [ $(.$(ps)es.$(type)).find-replace $(properties) : $(psval) ] ;
- if ! $(prev)
- {
- set-generated-target-ps $(ps) : $(type) : $(properties) : $(psval) ;
- }
-}
-
-
-# Returns either prefix or suffix (as indicated by 'ps') that should be used
-# when generating a target of 'type' with the specified properties. Parameter
-# 'ps' can be either "prefix" or "suffix". If no prefix/suffix is specified for
-# 'type', returns prefix/suffix for base type, if any.
-#
-local rule generated-target-ps ( ps : type : property-set )
-{
- local result ;
- local found ;
- while $(type) && ! $(found)
- {
- result = [ $(.$(ps)es.$(type)).find $(property-set) ] ;
- # If the prefix/suffix is explicitly set to an empty string, we consider
- # prefix/suffix to be found. If we were not to compare with "", there
- # would be no way to specify an empty prefix/suffix.
- if $(result)-is-defined
- {
- found = true ;
- }
- type = $(.base.$(type)) ;
- }
- if $(result) = ""
- {
- result = ;
- }
- return $(result) ;
-}
-
-
-# Returns file type given its name. If there are several dots in filename, tries
-# each suffix. E.g. for name of "file.so.1.2" suffixes "2", "1", and "so" will
-# be tried.
-#
-rule type ( filename )
-{
- if [ os.name ] in NT CYGWIN
- {
- filename = $(filename:L) ;
- }
- local type ;
- while ! $(type) && $(filename:S)
- {
- local suffix = $(filename:S) ;
- type = $(.type$(suffix)) ;
- filename = $(filename:S=) ;
- }
- return $(type) ;
-}
-
-
-# Rule used to construct all main targets. Note that this rule gets imported
-# into the global namespace under different alias names and the exact target
-# type to construct is selected based on the alias used to actually invoke this
-# rule.
-#
-rule main-target-rule ( name : sources * : requirements * : default-build * :
- usage-requirements * )
-{
- # First discover the required target type based on the exact alias used to
- # invoke this rule.
- local bt = [ BACKTRACE 1 ] ;
- local rulename = $(bt[4]) ;
- local target-type = [ type-from-rule-name $(rulename) ] ;
-
- # This is a circular module dependency and so must be imported here.
- import targets ;
-
- return [ targets.create-typed-target $(target-type) : [ project.current ] :
- $(name) : $(sources) : $(requirements) : $(default-build) :
- $(usage-requirements) ] ;
-}
-
-
-rule __test__ ( )
-{
- import assert ;
-
- # TODO: Add tests for all the is-derived, is-base & related type relation
- # checking rules.
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/build/version.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/build/version.jam b/ext/kenlm/jam-files/boost-build/build/version.jam
deleted file mode 100644
index d4b787d..0000000
--- a/ext/kenlm/jam-files/boost-build/build/version.jam
+++ /dev/null
@@ -1,165 +0,0 @@
-# Copyright 2002, 2003, 2004, 2006 Vladimir Prus
-# Copyright 2008, 2012 Jurko Gospodnetic
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-import numbers ;
-
-
-.major = "2011" ;
-.minor = "12" ;
-
-
-rule boost-build ( )
-{
- return "$(.major).$(.minor)-svn" ;
-}
-
-
-rule print ( )
-{
- if [ verify-engine-version ]
- {
- ECHO "Boost.Build" [ boost-build ] ;
- }
-}
-
-
-rule verify-engine-version ( )
-{
- local v = [ modules.peek : JAM_VERSION ] ;
-
- if $(v[1]) != $(.major) || $(v[2]) != $(.minor)
- {
- local argv = [ modules.peek : ARGV ] ;
- local e = $(argv[1]) ;
- local l = [ modules.binding version ] ;
- l = $(l:D) ;
- l = $(l:D) ;
- ECHO "warning: mismatched versions of Boost.Build engine and core" ;
- ECHO "warning: Boost.Build engine ($(e)) is $(v:J=.)" ;
- ECHO "warning: Boost.Build core (at $(l)) is" [ boost-build ] ;
- }
- else
- {
- return true ;
- }
-}
-
-
-# Utility rule for testing whether all elements in a sequence are equal to 0.
-#
-local rule is-all-zeroes ( sequence * )
-{
- local result = "true" ;
- for local e in $(sequence)
- {
- if $(e) != "0"
- {
- result = "" ;
- }
- }
- return $(result) ;
-}
-
-
-# Returns "true" if the first version is less than the second one.
-#
-rule version-less ( lhs + : rhs + )
-{
- numbers.check $(lhs) ;
- numbers.check $(rhs) ;
-
- local done ;
- local result ;
-
- while ! $(done) && $(lhs) && $(rhs)
- {
- if [ numbers.less $(lhs[1]) $(rhs[1]) ]
- {
- done = "true" ;
- result = "true" ;
- }
- else if [ numbers.less $(rhs[1]) $(lhs[1]) ]
- {
- done = "true" ;
- }
- else
- {
- lhs = $(lhs[2-]) ;
- rhs = $(rhs[2-]) ;
- }
- }
- if ( ! $(done) && ! $(lhs) && ! [ is-all-zeroes $(rhs) ] )
- {
- result = "true" ;
- }
-
- return $(result) ;
-}
-
-
-# Returns "true" if the current JAM version version is at least the given
-# version.
-#
-rule check-jam-version ( version + )
-{
- local version-tag = $(version:J=.) ;
- if ! $(version-tag)
- {
- import errors ;
- errors.error Invalid version specifier: : $(version:E="(undefined)") ;
- }
-
- if ! $(.jam-version-check.$(version-tag))-is-defined
- {
- local jam-version = [ modules.peek : JAM_VERSION ] ;
- if ! $(jam-version)
- {
- import errors ;
- errors.error "Unable to deduce Boost Jam version. Your Boost Jam"
- "installation is most likely terribly outdated." ;
- }
- .jam-version-check.$(version-tag) = "true" ;
- if [ version-less [ modules.peek : JAM_VERSION ] : $(version) ]
- {
- .jam-version-check.$(version-tag) = "" ;
- }
- }
- return $(.jam-version-check.$(version-tag)) ;
-}
-
-
-rule __test__ ( )
-{
- import assert ;
-
- local jam-version = [ modules.peek : JAM_VERSION ] ;
- local future-version = $(jam-version) ;
- future-version += "1" ;
-
- assert.true check-jam-version $(jam-version) ;
- assert.false check-jam-version $(future-version) ;
-
- assert.true version-less 0 : 1 ;
- assert.false version-less 0 : 0 ;
- assert.true version-less 1 : 2 ;
- assert.false version-less 1 : 1 ;
- assert.false version-less 2 : 1 ;
- assert.true version-less 3 1 20 : 3 4 10 ;
- assert.false version-less 3 1 10 : 3 1 10 ;
- assert.false version-less 3 4 10 : 3 1 20 ;
- assert.true version-less 3 1 20 5 1 : 3 4 10 ;
- assert.false version-less 3 1 10 5 1 : 3 1 10 ;
- assert.false version-less 3 4 10 5 1 : 3 1 20 ;
- assert.true version-less 3 1 20 : 3 4 10 5 1 ;
- assert.true version-less 3 1 10 : 3 1 10 5 1 ;
- assert.false version-less 3 4 10 : 3 1 20 5 1 ;
- assert.false version-less 3 1 10 : 3 1 10 0 0 ;
- assert.false version-less 3 1 10 0 0 : 3 1 10 ;
- assert.false version-less 3 1 10 0 : 3 1 10 0 0 ;
- assert.false version-less 3 1 10 0 : 03 1 10 0 0 ;
- assert.false version-less 03 1 10 0 : 3 1 10 0 0 ;
-
- # TODO: Add tests for invalid input data being sent to version-less.
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/build/virtual-target.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/build/virtual-target.jam b/ext/kenlm/jam-files/boost-build/build/virtual-target.jam
deleted file mode 100644
index f62eadb..0000000
--- a/ext/kenlm/jam-files/boost-build/build/virtual-target.jam
+++ /dev/null
@@ -1,1339 +0,0 @@
-# Copyright 2003 Dave Abrahams
-# Copyright 2005, 2006 Rene Rivera
-# Copyright 2002, 2003, 2004, 2005, 2006 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# Implements virtual targets, which correspond to actual files created during a
-# build, but are not yet targets in Jam sense. They are needed, for example,
-# when searching for possible transformation sequences, when it is not yet known
-# whether a particular target should be created at all.
-#
-# +--------------------------+
-# | virtual-target |
-# +==========================+
-# | actualize |
-# +--------------------------+
-# | actualize-action() = 0 |
-# | actualize-location() = 0 |
-# +----------------+---------+
-# |
-# ^
-# / \
-# +-+-+
-# |
-# +---------------------+ +-------+--------------+
-# | action | | abstract-file-target |
-# +=====================| * +======================+
-# | action-name | +--+ action |
-# | properties | | +----------------------+
-# +---------------------+--+ | actualize-action() |
-# | actualize() |0..1 +-----------+----------+
-# | path() | |
-# | adjust-properties() | sources |
-# | actualize-sources() | targets |
-# +------+--------------+ ^
-# | / \
-# ^ +-+-+
-# / \ |
-# +-+-+ +-------------+-------------+
-# | | |
-# | +------+---------------+ +--------+-------------+
-# | | file-target | | searched-lib-target |
-# | +======================+ +======================+
-# | | actualize-location() | | actualize-location() |
-# | +----------------------+ +----------------------+
-# |
-# +-+------------------------------+
-# | |
-# +----+----------------+ +---------+-----------+
-# | compile-action | | link-action |
-# +=====================+ +=====================+
-# | adjust-properties() | | adjust-properties() |
-# +---------------------+ | actualize-sources() |
-# +---------------------+
-#
-# The 'compile-action' and 'link-action' classes are not defined here but in
-# builtin.jam modules. They are shown in the diagram to give the big picture.
-
-import "class" : new ;
-import path ;
-import sequence ;
-import set ;
-import type ;
-import utility ;
-
-
-# Models a potential target. It can be converted into a Jam target and used in
-# building, if needed. However, it can be also dropped, which allows us to
-# search for different transformations and select only one.
-#
-class virtual-target
-{
- import scanner ;
- import sequence ;
- import utility ;
- import virtual-target ;
-
- rule __init__ (
- name # Target/project name.
- : project # Project to which this target belongs.
- )
- {
- self.name = $(name) ;
- self.project = $(project) ;
- self.dependencies = ;
- }
-
- # Name of this target.
- #
- rule name ( )
- {
- return $(self.name) ;
- }
-
- # Project of this target.
- #
- rule project ( )
- {
- return $(self.project) ;
- }
-
- # Adds additional 'virtual-target' instances this one depends on.
- #
- rule depends ( d + )
- {
- self.dependencies = [ sequence.merge $(self.dependencies) :
- [ sequence.insertion-sort $(d) ] ] ;
- }
-
- rule dependencies ( )
- {
- return $(self.dependencies) ;
- }
-
- rule always ( )
- {
- .always = 1 ;
- }
-
- # Generates all the actual targets and sets up build actions for this
- # target.
- #
- # If 'scanner' is specified, creates an additional target with the same
- # location as the actual target, which will depend on the actual target and
- # be associated with a 'scanner'. That additional target is returned. See
- # the docs (#dependency_scanning) for rationale. Target must correspond to a
- # file if 'scanner' is specified.
- #
- # If scanner is not specified then the actual target is returned.
- #
- rule actualize ( scanner ? )
- {
- local actual-name = [ actualize-no-scanner ] ;
-
- if $(.always)
- {
- ALWAYS $(actual-name) ;
- }
-
- if ! $(scanner)
- {
- return $(actual-name) ;
- }
- else
- {
- # Add the scanner instance to the grist for name.
- local g = [ sequence.join [ utility.ungrist $(actual-name:G) ]
- $(scanner) : - ] ;
- local name = $(actual-name:G=$(g)) ;
-
- if ! $(self.made.$(scanner))
- {
- self.made.$(scanner) = true ;
- actualize-location $(name) ;
- scanner.install $(scanner) : $(name) ;
- }
- return $(name) ;
- }
- }
-
-# private: (overridables)
-
- # Sets up build actions for 'target'. Should call appropriate rules and set
- # target variables.
- #
- rule actualize-action ( target )
- {
- import errors : error : errors.error ;
- errors.error "method should be defined in derived classes" ;
- }
-
- # Sets up variables on 'target' which specify its location.
- #
- rule actualize-location ( target )
- {
- import errors : error : errors.error ;
- errors.error "method should be defined in derived classes" ;
- }
-
- # If the target is a generated one, returns the path where it will be
- # generated. Otherwise, returns an empty list.
- #
- rule path ( )
- {
- import errors : error : errors.error ;
- errors.error "method should be defined in derived classes" ;
- }
-
- # Returns the actual target name to be used in case when no scanner is
- # involved.
- #
- rule actual-name ( )
- {
- import errors : error : errors.error ;
- errors.error "method should be defined in derived classes" ;
- }
-
-# implementation
- rule actualize-no-scanner ( )
- {
- # In fact, we just need to merge virtual-target with
- # abstract-file-target as the latter is the only class derived from the
- # former. But that has been left for later.
-
- import errors : error : errors.error ;
- errors.error "method should be defined in derived classes" ;
- }
-}
-
-
-# Target corresponding to a file. The exact mapping for file is not yet
-# specified in this class. (TODO: Actually, the class name could be better...)
-#
-# May be a source file (when no action is specified) or a derived file
-# (otherwise).
-#
-# The target's grist is a concatenation of its project's location, action
-# properties (for derived targets) and, optionally, value identifying the main
-# target.
-#
-class abstract-file-target : virtual-target
-{
- import project ;
- import regex ;
- import sequence ;
- import path ;
- import type ;
- import property-set ;
- import indirect ;
-
- rule __init__ (
- name # Target's name.
- exact ? # If non-empty, the name is exactly the name created file
- # should have. Otherwise, the '__init__' method will add a
- # suffix obtained from 'type' by calling
- # 'type.generated-target-suffix'.
- : type ? # Target's type.
- : project
- : action ?
- )
- {
- virtual-target.__init__ $(name) : $(project) ;
-
- self.type = $(type) ;
- self.action = $(action) ;
- if $(action)
- {
- $(action).add-targets $(__name__) ;
-
- if $(self.type) && ! $(exact)
- {
- _adjust-name $(name) ;
- }
- }
- }
-
- rule type ( )
- {
- return $(self.type) ;
- }
-
- # Sets the path. When generating target name, it will override any path
- # computation from properties.
- #
- rule set-path ( path )
- {
- self.path = [ path.native $(path) ] ;
- }
-
- # Returns the currently set action.
- #
- rule action ( )
- {
- return $(self.action) ;
- }
-
- # Sets/gets the 'root' flag. Target is root if it directly corresponds to
- # some variant of a main target.
- #
- rule root ( set ? )
- {
- if $(set)
- {
- self.root = true ;
- }
- return $(self.root) ;
- }
-
- # Gets or sets the subvariant which created this target. Subvariant is set
- # when target is brought into existance and is never changed after that. In
- # particular, if a target is shared by multiple subvariants, only the first
- # one is stored.
- #
- rule creating-subvariant ( s ? # If specified, specifies the value to set,
- # which should be a 'subvariant' class
- # instance.
- )
- {
- if $(s) && ! $(self.creating-subvariant)
- {
- self.creating-subvariant = $(s) ;
- }
- return $(self.creating-subvariant) ;
- }
-
- rule actualize-action ( target )
- {
- if $(self.action)
- {
- $(self.action).actualize ;
- }
- }
-
- # Return a human-readable representation of this target. If this target has
- # an action, that is:
- #
- # { <action-name>-<self.name>.<self.type> <action-sources>... }
- #
- # otherwise, it is:
- #
- # { <self.name>.<self.type> }
- #
- rule str ( )
- {
- local action = [ action ] ;
- local name-dot-type = [ sequence.join $(self.name) "." $(self.type) ] ;
-
- if $(action)
- {
- local sources = [ $(action).sources ] ;
- local action-name = [ $(action).action-name ] ;
-
- local ss ;
- for local s in $(sources)
- {
- ss += [ $(s).str ] ;
- }
-
- return "{" $(action-name)-$(name-dot-type) $(ss) "}" ;
- }
- else
- {
- return "{" $(name-dot-type) "}" ;
- }
- }
-
- rule less ( a )
- {
- if [ str ] < [ $(a).str ]
- {
- return true ;
- }
- }
-
- rule equal ( a )
- {
- if [ str ] = [ $(a).str ]
- {
- return true ;
- }
- }
-
-# private:
- rule actual-name ( )
- {
- if ! $(self.actual-name)
- {
- local grist = [ grist ] ;
- local basename = [ path.native $(self.name) ] ;
- self.actual-name = <$(grist)>$(basename) ;
- }
- return $(self.actual-name) ;
- }
-
- # Helper to 'actual-name', above. Computes a unique prefix used to
- # distinguish this target from other targets with the same name creating
- # different files.
- #
- rule grist ( )
- {
- # Depending on target, there may be different approaches to generating
- # unique prefixes. We generate prefixes in the form:
- # <one letter approach code> <the actual prefix>
- local path = [ path ] ;
- if $(path)
- {
- # The target will be generated to a known path. Just use the path
- # for identification, since path is as unique as it can get.
- return p$(path) ;
- }
- else
- {
- # File is either source, which will be searched for, or is not a
- # file at all. Use the location of project for distinguishing.
- local project-location = [ $(self.project).get location ] ;
- local location-grist = [ sequence.join [ regex.split
- $(project-location) "/" ] : "!" ] ;
-
- if $(self.action)
- {
- local ps = [ $(self.action).properties ] ;
- local property-grist = [ $(ps).as-path ] ;
- # 'property-grist' can be empty when 'ps' is an empty property
- # set.
- if $(property-grist)
- {
- location-grist = $(location-grist)/$(property-grist) ;
- }
- }
-
- return l$(location-grist) ;
- }
- }
-
- # Given the target name specified in constructor, returns the name which
- # should be really used, by looking at the <tag> properties. Tag properties
- # need to be specified as <tag>@rule-name. This makes Boost Build call the
- # specified rule with the target name, type and properties to get the new
- # name. If no <tag> property is specified or the rule specified by <tag>
- # returns nothing, returns the result of calling
- # virtual-target.add-prefix-and-suffix.
- #
- rule _adjust-name ( specified-name )
- {
- local ps ;
- if $(self.action)
- {
- ps = [ $(self.action).properties ] ;
- }
- else
- {
- ps = [ property-set.empty ] ;
- }
-
- local tag = [ $(ps).get <tag> ] ;
-
- if $(tag)
- {
- local rule-name = [ MATCH ^@(.*) : $(tag) ] ;
- if $(rule-name)
- {
- if $(tag[2])
- {
- import errors : error : errors.error ;
- errors.error <tag>@rulename is present but is not the only
- <tag> feature. ;
- }
-
- self.name = [ indirect.call $(rule-name) $(specified-name)
- : $(self.type) : $(ps) ] ;
- }
- else
- {
- import errors : error : errors.error ;
- errors.error <tag> property value must be '@rule-name'. ;
- }
- }
-
- # If there is no tag or the tag rule returned nothing.
- if ! $(tag) || ! $(self.name)
- {
- self.name = [ virtual-target.add-prefix-and-suffix $(specified-name)
- : $(self.type) : $(ps) ] ;
- }
- }
-
- rule actualize-no-scanner ( )
- {
- local name = [ actual-name ] ;
-
- # Do anything only on the first invocation.
- if ! $(self.made-no-scanner)
- {
- self.made-no-scanner = true ;
-
- if $(self.action)
- {
- # For non-derived target, we do not care if there are several
- # virtual targets that refer to the same name. One case when
- # this is unavoidable is when the file name is main.cpp and two
- # targets have types CPP (for compiling) and MOCCABLE_CPP (for
- # conversion to H via Qt tools).
- virtual-target.register-actual-name $(name) : $(__name__) ;
- }
-
- for local i in $(self.dependencies)
- {
- DEPENDS $(name) : [ $(i).actualize ] ;
- }
-
- actualize-location $(name) ;
- actualize-action $(name) ;
- }
- return $(name) ;
- }
-}
-
-
-# Appends the suffix appropriate to 'type/property-set' combination to the
-# specified name and returns the result.
-#
-rule add-prefix-and-suffix ( specified-name : type ? : property-set )
-{
- local suffix = [ type.generated-target-suffix $(type) : $(property-set) ] ;
-
- # Handle suffixes for which no leading dot is desired. Those are specified
- # by enclosing them in <...>. Needed by python so it can create "_d.so"
- # extensions, for example.
- if $(suffix:G)
- {
- suffix = [ utility.ungrist $(suffix) ] ;
- }
- else
- {
- suffix = .$(suffix) ;
- }
-
- local prefix = [ type.generated-target-prefix $(type) : $(property-set) ] ;
-
- if [ MATCH ^($(prefix)) : $(specified-name) ]
- {
- prefix = ;
- }
- return $(prefix:E="")$(specified-name)$(suffix:E="") ;
-}
-
-
-# File targets with explicitly known location.
-#
-# The file path is determined as
-# * Value passed to the 'set-path' method, if any.
-# * For derived files, project's build dir, joined with components that
-# describe action properties. If free properties are not equal to the
-# project's reference properties an element with the name of the main
-# target is added.
-# * For source files, project's source dir.
-#
-# The file suffix is determined as:
-# * The value passed to the 'suffix' method, if any.
-# * The suffix corresponding to the target's type.
-#
-class file-target : abstract-file-target
-{
- import "class" : new ;
- import common ;
-
- rule __init__ (
- name exact ?
- : type ? # Optional type for this target.
- : project
- : action ?
- : path ?
- )
- {
- abstract-file-target.__init__ $(name) $(exact) : $(type) : $(project) :
- $(action) ;
-
- self.path = $(path) ;
- }
-
- rule clone-with-different-type ( new-type )
- {
- return [ new file-target $(self.name) exact : $(new-type) :
- $(self.project) : $(self.action) : $(self.path) ] ;
- }
-
- rule actualize-location ( target )
- {
- # Scanner targets are always bound to already existing files in already
- # existing folder. They need to be marked as depending on their base
- # target (i.e. the target being scanned) but, unlike regular
- # dependencies set up by the DEPENDS rule, they must not depend on any
- # targets already marked as included by the base target. Otherwise such
- # an included file being newer than the file being scanned would cause
- # the scanner target to be updated, further causing any target depending
- # on that scanner target to be rebuilt. This is the exact relationship
- # as set up by Boost Jam's SEARCH binding method (needed to support
- # searching for generated targets) so we want to bind scanner targets
- # using this method instead of explicitly specifying their location
- # using LOCATE.
- #
- # FIXME: We recognize scanner targets by their given name being
- # different from this target's actual name. This is a hack and should be
- # cleaned up by reorganizing who knows about scanners in the
- # virtual-target/abstract-file-target/file-target/notfile-target/
- # searched-lib-target/... class hierarchy.
- local is-scanner-target ;
- if $(target) != [ actual-name ]
- {
- is-scanner-target = true ;
- }
-
- if $(self.action) && ! $(is-scanner-target)
- {
- # This is a derived file.
- local path = [ path ] ;
- LOCATE on $(target) = $(path) ;
-
- # Make sure the path exists.
- DEPENDS $(target) : $(path) ;
- common.MkDir $(path) ;
-
- # It is possible that the target name includes a directory too, for
- # example when installing headers. Create that directory.
- if $(target:D)
- {
- local d = $(target:D) ;
- d = $(d:R=$(path)) ;
- DEPENDS $(target) : $(d) ;
- common.MkDir $(d) ;
- }
-
- # For a real file target, we create a fake target depending on the
- # real target. This allows us to run
- #
- # b2 hello.o
- #
- # without trying to guess the name of the real target. Note that the
- # target has no directory name and uses a special <e> grist.
- #
- # First, that means that "b2 hello.o" will build all known hello.o
- # targets. Second, the <e> grist makes sure this target will not be
- # confused with other targets, for example, if we have subdir 'test'
- # with target 'test' in it that includes a 'test.o' file, then the
- # target for directory will be just 'test' the target for test.o
- # will be <ptest/bin/gcc/debug>test.o and the target we create below
- # will be <e>test.o
- DEPENDS $(target:G=e) : $(target) ;
- # Allow b2 <path-to-file>/<file> to work. This will not catch all
- # possible ways to refer to the path (relative/absolute, extra ".",
- # various "..", but should help in obvious cases.
- DEPENDS $(target:G=e:R=$(path)) : $(target) ;
- }
- else
- {
- SEARCH on $(target) = [ path.native $(self.path) ] ;
- }
- }
-
- # Returns the directory for this target.
- #
- rule path ( )
- {
- if ! $(self.path)
- {
- if $(self.action)
- {
- local p = [ $(self.action).properties ] ;
- local path,relative-to-build-dir = [ $(p).target-path ] ;
- local path = $(path,relative-to-build-dir[1]) ;
- local relative-to-build-dir = $(path,relative-to-build-dir[2]) ;
-
- if $(relative-to-build-dir)
- {
- path = [ path.join [ $(self.project).build-dir ] $(path) ] ;
- }
-
- self.path = [ path.native $(path) ] ;
- }
- }
- return $(self.path) ;
- }
-}
-
-
-class notfile-target : abstract-file-target
-{
- rule __init__ ( name : project : action ? )
- {
- abstract-file-target.__init__ $(name) : : $(project) : $(action) ;
- }
-
- # Returns nothing to indicate that the target's path is not known.
- #
- rule path ( )
- {
- return ;
- }
-
- rule actualize-location ( target )
- {
- NOTFILE $(target) ;
- ALWAYS $(target) ;
- # TEMPORARY $(target) ;
- NOUPDATE $(target) ;
- }
-}
-
-
-# Class representing an action. Both 'targets' and 'sources' should list
-# instances of 'virtual-target'. Action name should name a rule with this
-# prototype:
-# rule action-name ( targets + : sources * : properties * )
-# Targets and sources are passed as actual Jam targets. The rule may not
-# establish additional dependency relationships.
-#
-class action
-{
- import "class" ;
- import indirect ;
- import path ;
- import property-set ;
- import set : difference ;
- import toolset ;
- import type ;
-
- rule __init__ ( sources * : action-name + : property-set ? )
- {
- self.sources = $(sources) ;
-
- self.action-name = [ indirect.make-qualified $(action-name) ] ;
-
- if ! $(property-set)
- {
- property-set = [ property-set.empty ] ;
- }
-
- if ! [ class.is-instance $(property-set) ]
- {
- import errors : error : errors.error ;
- errors.error "Property set instance required" ;
- }
-
- self.properties = $(property-set) ;
- }
-
- rule add-targets ( targets * )
- {
- self.targets += $(targets) ;
- }
-
- rule replace-targets ( old-targets * : new-targets * )
- {
- self.targets = [ set.difference $(self.targets) : $(old-targets) ] ;
- self.targets += $(new-targets) ;
- }
-
- rule targets ( )
- {
- return $(self.targets) ;
- }
-
- rule sources ( )
- {
- return $(self.sources) ;
- }
-
- rule action-name ( )
- {
- return $(self.action-name) ;
- }
-
- rule properties ( )
- {
- return $(self.properties) ;
- }
-
- # Generates actual build instructions.
- #
- rule actualize ( )
- {
- if ! $(self.actualized)
- {
- self.actualized = true ;
-
- local ps = [ properties ] ;
- local properties = [ adjust-properties $(ps) ] ;
-
- local actual-targets ;
- for local i in [ targets ]
- {
- actual-targets += [ $(i).actualize ] ;
- }
-
- actualize-sources [ sources ] : $(properties) ;
-
- DEPENDS $(actual-targets) : $(self.actual-sources)
- $(self.dependency-only-sources) ;
-
- # Action name can include additional rule arguments, which should
- # not be passed to 'set-target-variables'.
- toolset.set-target-variables
- [ indirect.get-rule $(self.action-name[1]) ] $(actual-targets)
- : $(properties) ;
-
- # Reflect ourselves in a variable for the target. This allows
- # looking up additional info for the action given the raw target.
- # For example to debug or output action information from action
- # rules.
- .action on $(actual-targets) = $(__name__) ;
-
- indirect.call $(self.action-name) $(actual-targets)
- : $(self.actual-sources) : [ $(properties).raw ] ;
-
- # Since we set up the creating action here, we set up the action for
- # cleaning up as well.
- common.Clean clean-all : $(actual-targets) ;
- }
- }
-
- # Helper for 'actualize-sources'. For each passed source, actualizes it with
- # the appropriate scanner. Returns the actualized virtual targets.
- #
- rule actualize-source-type ( sources * : property-set )
- {
- local result = ;
- for local i in $(sources)
- {
- local scanner ;
- if [ $(i).type ]
- {
- scanner = [ type.get-scanner [ $(i).type ] : $(property-set) ] ;
- }
- result += [ $(i).actualize $(scanner) ] ;
- }
- return $(result) ;
- }
-
- # Creates actual Jam targets for sources. Initializes the following member
- # variables:
- # 'self.actual-sources' -- sources passed to the updating action.
- # 'self.dependency-only-sources' -- sources marked as dependencies, but
- # are not used otherwise.
- #
- # New values will be *appended* to the variables. They may be non-empty if
- # caller wants it.
- #
- rule actualize-sources ( sources * : property-set )
- {
- local dependencies = [ $(self.properties).get <dependency> ] ;
-
- self.dependency-only-sources +=
- [ actualize-source-type $(dependencies) : $(property-set) ] ;
- self.actual-sources +=
- [ actualize-source-type $(sources) : $(property-set) ] ;
-
- # This is used to help b2 find dependencies in generated headers and
- # other main targets, e.g. in:
- #
- # make a.h : ....... ;
- # exe hello : hello.cpp : <implicit-dependency>a.h ;
- #
- # For b2 to find the dependency the generated target must be
- # actualized (i.e. have its Jam target constructed). In the above case,
- # if we are building just hello ("b2 hello"), 'a.h' will not be
- # actualized unless we do it here.
- local implicit = [ $(self.properties).get <implicit-dependency> ] ;
- for local i in $(implicit)
- {
- $(i:G=).actualize ;
- }
- }
-
- # Determines real properties when trying to build with 'properties'. This is
- # the last chance to fix properties, for example to adjust includes to get
- # generated headers correctly. Default implementation simply returns its
- # argument.
- #
- rule adjust-properties ( property-set )
- {
- return $(property-set) ;
- }
-}
-
-
-# Action class which does nothing --- it produces the targets with specific
-# properties out of nowhere. It is needed to distinguish virtual targets with
-# different properties that are known to exist and have no actions which create
-# them.
-#
-class null-action : action
-{
- rule __init__ ( property-set ? )
- {
- action.__init__ : .no-action : $(property-set) ;
- }
-
- rule actualize ( )
- {
- if ! $(self.actualized)
- {
- self.actualized = true ;
- for local i in [ targets ]
- {
- $(i).actualize ;
- }
- }
- }
-}
-
-
-# Class which acts exactly like 'action', except that its sources are not
-# scanned for dependencies.
-#
-class non-scanning-action : action
-{
- rule __init__ ( sources * : action-name + : property-set ? )
- {
- action.__init__ $(sources) : $(action-name) : $(property-set) ;
- }
-
- rule actualize-source-type ( sources * : property-set )
- {
- local result ;
- for local i in $(sources)
- {
- result += [ $(i).actualize ] ;
- }
- return $(result) ;
- }
-}
-
-
-# Creates a virtual target with an appropriate name and type from 'file'. If a
-# target with that name in that project already exists, returns that already
-# created target.
-#
-# FIXME: a more correct way would be to compute the path to the file, based on
-# name and source location for the project, and use that path to determine if
-# the target has already been created. This logic should be shared with how we
-# usually find targets identified by a specific target id. It should also be
-# updated to work correctly when the file is specified using both relative and
-# absolute paths.
-#
-# TODO: passing a project with all virtual targets is starting to be annoying.
-#
-rule from-file ( file : file-loc : project )
-{
- import type ; # Had to do this here to break a circular dependency.
-
- # Check whether we already created a target corresponding to this file.
- local path = [ path.root [ path.root $(file) $(file-loc) ] [ path.pwd ] ] ;
-
- if $(.files.$(path))
- {
- return $(.files.$(path)) ;
- }
- else
- {
- local name = [ path.make $(file) ] ;
- local type = [ type.type $(file) ] ;
- local result ;
-
- result = [ new file-target $(file) : $(type) : $(project) : :
- $(file-loc) ] ;
-
- .files.$(path) = $(result) ;
- return $(result) ;
- }
-}
-
-
-# Registers a new virtual target. Checks if there is already a registered target
-# with the same name, type, project and subvariant properties as well as the
-# same sources and equal action. If such target is found it is returned and a
-# new 'target' is not registered. Otherwise, 'target' is registered and
-# returned.
-#
-rule register ( target )
-{
- local signature = [ sequence.join [ $(target).path ] [ $(target).name ] : -
- ] ;
-
- local result ;
- for local t in $(.cache.$(signature))
- {
- local a1 = [ $(t).action ] ;
- local a2 = [ $(target).action ] ;
-
- if ! $(result)
- {
- if ! $(a1) && ! $(a2)
- {
- result = $(t) ;
- }
- else if $(a1) && $(a2) &&
- ( [ $(a1).action-name ] = [ $(a2).action-name ] ) &&
- ( [ $(a1).sources ] = [ $(a2).sources ] )
- {
- local ps1 = [ $(a1).properties ] ;
- local ps2 = [ $(a2).properties ] ;
- local p1 = [ $(ps1).base ] [ $(ps1).free ] [ set.difference
- [ $(ps1).dependency ] : [ $(ps1).incidental ] ] ;
- local p2 = [ $(ps2).base ] [ $(ps2).free ] [ set.difference
- [ $(ps2).dependency ] : [ $(ps2).incidental ] ] ;
- if $(p1) = $(p2)
- {
- result = $(t) ;
- }
- }
- }
- }
-
- if ! $(result)
- {
- .cache.$(signature) += $(target) ;
- result = $(target) ;
- }
-
- .recent-targets += $(result) ;
- .all-targets += $(result) ;
-
- return $(result) ;
-}
-
-
-# Each target returned by 'register' is added to the .recent-targets list,
-# returned by this function. This allows us to find all virtual targets created
-# when building a specific main target, even those constructed only as
-# intermediate targets.
-#
-rule recent-targets ( )
-{
- return $(.recent-targets) ;
-}
-
-
-rule clear-recent-targets ( )
-{
- .recent-targets = ;
-}
-
-
-# Returns all virtual targets ever created.
-#
-rule all-targets ( )
-{
- return $(.all-targets) ;
-}
-
-
-# Returns all targets from 'targets' with types equal to 'type' or derived from
-# it.
-#
-rule select-by-type ( type : targets * )
-{
- local result ;
- for local t in $(targets)
- {
- if [ type.is-subtype [ $(t).type ] $(type) ]
- {
- result += $(t) ;
- }
- }
- return $(result) ;
-}
-
-
-rule register-actual-name ( actual-name : virtual-target )
-{
- if $(.actual.$(actual-name))
- {
- local cs1 = [ $(.actual.$(actual-name)).creating-subvariant ] ;
- local cmt1-name ;
- if $(cs1)-is-defined
- {
- local cmt1 = [ $(cs1).main-target ] ;
- cmt1-name = [ $(cmt1).full-name ] ;
- }
- local cs2 = [ $(virtual-target).creating-subvariant ] ;
- local cmt2-name ;
- if $(cs2)-is-defined
- {
- local cmt2 = [ $(cs2).main-target ] ;
- cmt2-name = [ $(cmt2).full-name ] ;
- }
- local extra-error-information ;
- if ! $(cs1)-is-defined || ! $(cs2)-is-defined
- {
- extra-error-information = Encountered a virtual-target without a
- creating subvariant. It could be the virtual target has not been
- registered via the virtual-target.register rule. ;
- }
-
- local action1 = [ $(.actual.$(actual-name)).action ] ;
- local action2 = [ $(virtual-target).action ] ;
- local properties-added ;
- local properties-removed ;
- if $(action1) && $(action2)
- {
- local p1 = [ $(action1).properties ] ;
- p1 = [ $(p1).raw ] ;
- local p2 = [ $(action2).properties ] ;
- p2 = [ $(p2).raw ] ;
- properties-removed = [ set.difference $(p1) : $(p2) ] ;
- properties-removed ?= "none" ;
- properties-added = [ set.difference $(p2) : $(p1) ] ;
- properties-added ?= "none" ;
- }
- import errors : error : errors.error ;
- errors.error "Duplicate name of actual target:" $(actual-name)
- : "previous virtual target" [ $(.actual.$(actual-name)).str ]
- : "created from" $(cmt1-name)
- : "another virtual target" [ $(virtual-target).str ]
- : "created from" $(cmt2-name)
- : "added properties:" $(properties-added)
- : "removed properties:" $(properties-removed)
- : $(extra-error-information) ;
- }
- else
- {
- .actual.$(actual-name) = $(virtual-target) ;
- }
-}
-
-
-# Traverses the dependency graph of 'target' and return all targets that will be
-# created before this one is created. If the root of some dependency graph is
-# found during traversal, it is either included or not, depending on the
-# 'include-roots' value. In either case traversal stops at root targets, i.e.
-# root target sources are not traversed.
-#
-rule traverse ( target : include-roots ? : include-sources ? )
-{
- local result ;
- if [ $(target).action ]
- {
- local action = [ $(target).action ] ;
- # This includes the 'target' as well.
- result += [ $(action).targets ] ;
-
- for local t in [ $(action).sources ]
- {
- if ! [ $(t).root ]
- {
- result += [ traverse $(t) : $(include-roots) :
- $(include-sources) ] ;
- }
- else if $(include-roots)
- {
- result += $(t) ;
- }
- }
- }
- else if $(include-sources)
- {
- result = $(target) ;
- }
- return $(result) ;
-}
-
-
-# Takes an 'action' instance and creates a new instance of it and all targets
-# produced by the action. The rule-name and properties are set to
-# 'new-rule-name' and 'new-properties', if those are specified. Returns the
-# cloned action.
-#
-rule clone-action ( action : new-project : new-action-name ? : new-properties ?
- )
-{
- if ! $(new-action-name)
- {
- new-action-name = [ $(action).action-name ] ;
- }
- if ! $(new-properties)
- {
- new-properties = [ $(action).properties ] ;
- }
-
- local action-class = [ modules.peek $(action) : __class__ ] ;
- local cloned-action = [ class.new $(action-class)
- [ $(action).sources ] : $(new-action-name) : $(new-properties) ] ;
-
- local cloned-targets ;
- for local target in [ $(action).targets ]
- {
- local n = [ $(target).name ] ;
- # Do not modify produced target names.
- local cloned-target = [ class.new file-target $(n) exact :
- [ $(target).type ] : $(new-project) : $(cloned-action) ] ;
- local d = [ $(target).dependencies ] ;
- if $(d)
- {
- $(cloned-target).depends $(d) ;
- }
- $(cloned-target).root [ $(target).root ] ;
- $(cloned-target).creating-subvariant [ $(target).creating-subvariant ] ;
-
- cloned-targets += $(cloned-target) ;
- }
-
- return $(cloned-action) ;
-}
-
-
-class subvariant
-{
- import sequence ;
- import type ;
-
- rule __init__ ( main-target # The instance of main-target class.
- : property-set # Properties requested for this target.
- : sources *
- : build-properties # Actually used properties.
- : sources-usage-requirements # Properties propagated from sources.
- : created-targets * ) # Top-level created targets.
- {
- self.main-target = $(main-target) ;
- self.properties = $(property-set) ;
- self.sources = $(sources) ;
- self.build-properties = $(build-properties) ;
- self.sources-usage-requirements = $(sources-usage-requirements) ;
- self.created-targets = $(created-targets) ;
-
- # Pre-compose a list of other dependency graphs this one depends on.
- local deps = [ $(build-properties).get <implicit-dependency> ] ;
- for local d in $(deps)
- {
- self.other-dg += [ $(d:G=).creating-subvariant ] ;
- }
-
- self.other-dg = [ sequence.unique $(self.other-dg) ] ;
- }
-
- rule main-target ( )
- {
- return $(self.main-target) ;
- }
-
- rule created-targets ( )
- {
- return $(self.created-targets) ;
- }
-
- rule requested-properties ( )
- {
- return $(self.properties) ;
- }
-
- rule build-properties ( )
- {
- return $(self.build-properties) ;
- }
-
- rule sources-usage-requirements ( )
- {
- return $(self.sources-usage-requirements) ;
- }
-
- rule set-usage-requirements ( usage-requirements )
- {
- self.usage-requirements = $(usage-requirements) ;
- }
-
- rule usage-requirements ( )
- {
- return $(self.usage-requirements) ;
- }
-
- # Returns all targets referenced by this subvariant, either directly or
- # indirectly, and either as sources, or as dependency properties. Targets
- # referred to using the dependency property are returned as properties, not
- # targets.
- #
- rule all-referenced-targets ( theset )
- {
- # Find directly referenced targets.
- local deps = [ $(self.build-properties).dependency ] ;
- local all-targets = $(self.sources) $(deps) ;
-
- # Find other subvariants.
- local r ;
- for local t in $(all-targets)
- {
- if ! [ $(theset).contains $(t) ]
- {
- $(theset).add $(t) ;
- r += [ $(t:G=).creating-subvariant ] ;
- }
- }
- r = [ sequence.unique $(r) ] ;
- for local s in $(r)
- {
- if $(s) != $(__name__)
- {
- $(s).all-referenced-targets $(theset) ;
- }
- }
- }
-
- # Returns the properties specifying implicit include paths to generated
- # headers. This traverses all targets in this subvariant and subvariants
- # referred by <implicit-dependecy> properties. For all targets of type
- # 'target-type' (or for all targets, if 'target-type' is not specified), the
- # result will contain <$(feature)>path-to-that-target.
- #
- rule implicit-includes ( feature : target-type ? )
- {
- local key = ii$(feature)-$(target-type:E="") ;
- if ! $($(key))-is-not-empty
- {
- local target-paths = [ all-target-directories $(target-type) ] ;
- target-paths = [ sequence.unique $(target-paths) ] ;
- local result = $(target-paths:G=$(feature)) ;
- if ! $(result)
- {
- result = "" ;
- }
- $(key) = $(result) ;
- }
- if $($(key)) = ""
- {
- return ;
- }
- else
- {
- return $($(key)) ;
- }
- }
-
- rule all-target-directories ( target-type ? )
- {
- if ! $(self.target-directories)
- {
- compute-target-directories $(target-type) ;
- }
- return $(self.target-directories) ;
- }
-
- rule compute-target-directories ( target-type ? )
- {
- local result ;
- for local t in $(self.created-targets)
- {
- # Skip targets of the wrong type.
- if ! $(target-type) ||
- [ type.is-derived [ $(t).type ] $(target-type) ]
- {
- result = [ sequence.merge $(result) : [ $(t).path ] ] ;
- }
- }
- for local d in $(self.other-dg)
- {
- result += [ $(d).all-target-directories $(target-type) ] ;
- }
- self.target-directories = $(result) ;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/kernel/boost-build.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/kernel/boost-build.jam b/ext/kenlm/jam-files/boost-build/kernel/boost-build.jam
deleted file mode 100644
index 377f6ec..0000000
--- a/ext/kenlm/jam-files/boost-build/kernel/boost-build.jam
+++ /dev/null
@@ -1,5 +0,0 @@
-# Copyright 2003 Dave Abrahams
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-boost-build . ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/kernel/bootstrap.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/kernel/bootstrap.jam b/ext/kenlm/jam-files/boost-build/kernel/bootstrap.jam
deleted file mode 100644
index c4320dc..0000000
--- a/ext/kenlm/jam-files/boost-build/kernel/bootstrap.jam
+++ /dev/null
@@ -1,266 +0,0 @@
-# Copyright 2003 Dave Abrahams
-# Copyright 2003, 2005, 2006 Rene Rivera
-# Copyright 2003, 2005, 2006 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# First of all, check the jam version.
-if $(JAM_VERSION:J="") < 030112
-{
- ECHO "error: Boost.Jam version 3.1.12 or later required" ;
- EXIT ;
-}
-
-local required-rules = GLOB-RECURSIVELY HAS_NATIVE_RULE ;
-for local r in $(required-rules)
-{
- if ! $(r) in [ RULENAMES ]
- {
- ECHO "error: builtin rule '$(r)' is not present" ;
- ECHO "error: your version of bjam is likely out of date" ;
- ECHO "error: please get a fresh version from SVN." ;
- EXIT ;
- }
-}
-
-local native = regex transform 2 ;
-while $(native)
-{
- if ! [ HAS_NATIVE_RULE $(native[1]) : $(native[2]) : $(native[3]) ]
- {
- ECHO "error: missing native rule '$(native[1]).$(native[2])'" ;
- ECHO "error: or interface version of that rule is too low" ;
- ECHO "error: your version of bjam is likely out of date" ;
- ECHO "error: please get a fresh version from SVN." ;
- EXIT ;
- }
- native = $(native[4-]) ;
-}
-
-
-# Check that the builtin .ENVIRON module is present. We do not have a builtin to
-# check that a module is present, so we assume that the PATH environment
-# variable is always set and verify that the .ENVIRON module has a non-empty
-# value of that variable.
-module .ENVIRON
-{
- local p = $(PATH) $(Path) $(path) ;
- if ! $(p)
- {
- ECHO "error: no builtin module .ENVIRON is found" ;
- ECHO "error: your version of bjam is likely out of date" ;
- ECHO "error: please get a fresh version from SVN." ;
- EXIT ;
- }
-}
-
-# Check that @() functionality is present. Similarly to modules, we do not have
-# a way to test this directly. Instead we check that $(TMPNAME) functionality is
-# present which was added at roughly the same time (more precisely, it was added
-# just before).
-{
- if ! $(TMPNAME)
- {
- ECHO "error: no @() functionality found" ;
- ECHO "error: your version of b2 is likely out of date" ;
- ECHO "error: please get a fresh version from SVN." ;
- EXIT ;
- }
-}
-
-# Make sure that \n escape is avaiable.
-if "\n" = "n"
-{
- if $(OS) = CYGWIN
- {
- ECHO "warning: escape sequences are not supported" ;
- ECHO "warning: this will cause major misbehaviour on cygwin" ;
- ECHO "warning: your version of b2 is likely out of date" ;
- ECHO "warning: please get a fresh version from SVN." ;
- }
-}
-
-
-# Bootstrap the module system. Then bring the import rule into the global module.
-#
-SEARCH on <module@>modules.jam = $(.bootstrap-file:D) ;
-module modules { include <module@>modules.jam ; }
-IMPORT modules : import : : import ;
-
-{
- # Add module subdirectories to the BOOST_BUILD_PATH, which allows us to make
- # incremental refactoring steps by moving modules to appropriate
- # subdirectories, thereby achieving some physical separation of different
- # layers without changing all of our code to specify subdirectories in
- # import statements or use an extra level of qualification on imported
- # names.
-
- local subdirs =
- kernel # only the most-intrinsic modules: modules, errors
- util # low-level substrate: string/number handling, etc.
- build # essential elements of the build system architecture
- tools # toolsets for handling specific build jobs and targets.
- contrib # user contributed (unreviewed) modules
- . # build-system.jam lives here
- ;
- local whereami = [ NORMALIZE_PATH $(.bootstrap-file:DT) ] ;
- BOOST_BUILD_PATH += $(whereami:D)/$(subdirs) ;
-
- modules.poke .ENVIRON : BOOST_BUILD_PATH : $(BOOST_BUILD_PATH) ;
-
- modules.poke : EXTRA_PYTHONPATH : $(whereami) ;
-}
-
-# Reload the modules, to clean up things. The modules module can tolerate being
-# imported twice.
-#
-import modules ;
-
-# Process option plugins first to allow them to prevent loading the rest of the
-# build system.
-#
-import option ;
-local dont-build = [ option.process ] ;
-
-# Should we skip building, i.e. loading the build system, according to the
-# options processed?
-#
-if ! $(dont-build)
-{
- if ! --python in $(ARGV)
- {
- # Allow users to override the build system file from the command-line
- # (mostly for testing).
- local build-system = [ MATCH --build-system=(.*) : $(ARGV) ] ;
- build-system ?= build-system ;
-
- # Use last element in case of multiple command-line options.
- import $(build-system[-1]) ;
- }
- else
- {
- ECHO "Boost.Build V2 Python port (experimental)" ;
-
- # Define additional interface exposed to Python code. Python code will
- # also have access to select bjam builtins in the 'bjam' module, but
- # some things are easier to define outside C.
- module python_interface
- {
- rule load ( module-name : location )
- {
- USER_MODULE $(module-name) ;
- # Make all rules in the loaded module available in the global
- # namespace, so that we do not have to bother specifying the
- # "correct" module when calling from Python.
- module $(module-name)
- {
- __name__ = $(1) ;
- include $(2) ;
- local rules = [ RULENAMES $(1) ] ;
- IMPORT $(1) : $(rules) : $(1) : $(1).$(rules) ;
- }
- }
-
- rule peek ( module-name ? : variables + )
- {
- module $(<)
- {
- return $($(>)) ;
- }
- }
-
- rule set-variable ( module-name : name : value * )
- {
- module $(<)
- {
- $(>) = $(3) ;
- }
- }
-
- rule set-top-level-targets ( targets * )
- {
- DEPENDS all : $(targets) ;
- }
-
- rule call-in-module ( m : rulename : * )
- {
- module $(m)
- {
- return [ $(2) $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9)
- : $(10) : $(11) : $(12) : $(13) : $(14) : $(15) : $(16)
- : $(17) : $(18) : $(19) ] ;
- }
- }
-
-
- rule set-update-action ( action : targets * : sources * :
- properties * )
- {
- $(action) $(targets) : $(sources) : $(properties) ;
- }
-
- rule set-update-action-in-module ( m : action : targets * :
- sources * : properties * )
- {
- module $(m)
- {
- $(2) $(3) : $(4) : $(5) ;
- }
- }
-
- rule set-target-variable ( targets + : variable : value * : append ?
- )
- {
- if $(append)
- {
- $(variable) on $(targets) += $(value) ;
- }
- else
- {
- $(variable) on $(targets) = $(value) ;
- }
- }
-
- rule get-target-variable ( targets + : variable )
- {
- return [ on $(targets) return $($(variable)) ] ;
- }
-
- rule import-rules-from-parent ( parent-module : this-module :
- user-rules * )
- {
- IMPORT $(parent-module) : $(user-rules) : $(this-module) :
- $(user-rules) ;
- EXPORT $(this-module) : $(user-rules) ;
- }
-
- rule mark-included ( targets * : includes * )
- {
- NOCARE $(includes) ;
- INCLUDES $(targets) : $(includes) ;
- ISFILE $(includes) ;
- }
- }
-
- PYTHON_IMPORT_RULE bootstrap : bootstrap : PyBB : bootstrap ;
- modules.poke PyBB : root : [ NORMALIZE_PATH $(.bootstrap-file:DT)/.. ] ;
-
- module PyBB
- {
- local ok = [ bootstrap $(root) ] ;
- if ! $(ok)
- {
- EXIT ;
- }
- }
-
-
- #PYTHON_IMPORT_RULE boost.build.build_system : main : PyBB : main ;
-
- #module PyBB
- #{
- # main ;
- #}
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/kernel/class.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/kernel/class.jam b/ext/kenlm/jam-files/boost-build/kernel/class.jam
deleted file mode 100644
index e48ab6d..0000000
--- a/ext/kenlm/jam-files/boost-build/kernel/class.jam
+++ /dev/null
@@ -1,420 +0,0 @@
-# Copyright 2001, 2002, 2003 Dave Abrahams
-# Copyright 2002, 2005 Rene Rivera
-# Copyright 2002, 2003 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# Polymorphic class system built on top of core Jam facilities.
-#
-# Classes are defined by 'class' keywords:
-#
-# class myclass
-# {
-# rule __init__ ( arg1 ) # constructor
-# {
-# self.attribute = $(arg1) ;
-# }
-#
-# rule method1 ( ) # method
-# {
-# return [ method2 ] ;
-# }
-#
-# rule method2 ( ) # method
-# {
-# return $(self.attribute) ;
-# }
-# }
-#
-# The __init__ rule is the constructor, and sets member variables.
-#
-# New instances are created by invoking [ new <class> <args...> ]:
-#
-# local x = [ new myclass foo ] ; # x is a new myclass object
-# assert.result foo : [ $(x).method1 ] ; # $(x).method1 returns "foo"
-#
-# Derived class are created by mentioning base classes in the declaration::
-#
-# class derived : myclass
-# {
-# rule __init__ ( arg )
-# {
-# myclass.__init__ $(arg) ; # call base __init__
-#
-# }
-#
-# rule method2 ( ) # method override
-# {
-# return $(self.attribute)XXX ;
-# }
-# }
-#
-# All methods operate virtually, replacing behavior in the base classes. For
-# example::
-#
-# local y = [ new derived foo ] ; # y is a new derived object
-# assert.result fooXXX : [ $(y).method1 ] ; # $(y).method1 returns "foo"
-#
-# Each class instance is its own core Jam module. All instance attributes and
-# methods are accessible without additional qualification from within the class
-# instance. All rules imported in class declaration, or visible in base classses
-# are also visible. Base methods are available in qualified form:
-# base-name.method-name. By convention, attribute names are prefixed with
-# "self.".
-
-import modules ;
-import numbers ;
-
-
-rule xinit ( instance : class )
-{
- module $(instance)
- {
- __class__ = $(2) ;
- __name__ = $(1) ;
- }
-}
-
-
-rule new ( class args * : * )
-{
- .next-instance ?= 1 ;
- local id = object($(class))@$(.next-instance) ;
-
- INSTANCE $(id) : class@$(class) ;
- xinit $(id) : $(class) ;
- IMPORT_MODULE $(id) ;
- $(id).__init__ $(args) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) :
- $(9) : $(10) : $(11) : $(12) : $(13) : $(14) : $(15) : $(16) : $(17) :
- $(18) : $(19) ;
-
- # Bump the next unique object name.
- .next-instance = [ numbers.increment $(.next-instance) ] ;
-
- # Return the name of the new instance.
- return $(id) ;
-}
-
-
-rule bases ( class )
-{
- module class@$(class)
- {
- return $(__bases__) ;
- }
-}
-
-
-rule is-derived ( class : bases + )
-{
- local stack = $(class) ;
- local visited found ;
- while ! $(found) && $(stack)
- {
- local top = $(stack[1]) ;
- stack = $(stack[2-]) ;
- if ! ( $(top) in $(visited) )
- {
- visited += $(top) ;
- stack += [ bases $(top) ] ;
-
- if $(bases) in $(visited)
- {
- found = true ;
- }
- }
- }
- return $(found) ;
-}
-
-
-# Returns true if the 'value' is a class instance.
-#
-rule is-instance ( value )
-{
- return [ MATCH "^(object\\()[^@]+\\)@.*" : $(value) ] ;
-}
-
-
-# Check if the given value is of the given type.
-#
-rule is-a (
- instance # The value to check.
- : type # The type to test for.
-)
-{
- if [ is-instance $(instance) ]
- {
- return [ class.is-derived [ modules.peek $(instance) : __class__ ] : $(type) ] ;
- }
-}
-
-
-local rule typecheck ( x )
-{
- local class-name = [ MATCH "^\\[(.*)\\]$" : [ BACKTRACE 1 ] ] ;
- if ! [ is-a $(x) : $(class-name) ]
- {
- return "Expected an instance of "$(class-name)" but got \""$(x)"\" for argument" ;
- }
-}
-
-
-rule __test__ ( )
-{
- import assert ;
- import "class" : new ;
- import errors : try catch ;
-
- # This will be the construction function for a class called 'myclass'.
- #
- class myclass
- {
- import assert ;
-
- rule __init__ ( x_ * : y_ * )
- {
- # Set some instance variables.
- x = $(x_) ;
- y = $(y_) ;
- foo += 10 ;
- }
-
- rule set-x ( newx * )
- {
- x = $(newx) ;
- }
-
- rule get-x ( )
- {
- return $(x) ;
- }
-
- rule set-y ( newy * )
- {
- y = $(newy) ;
- }
-
- rule get-y ( )
- {
- return $(y) ;
- }
-
- rule f ( )
- {
- return [ g $(x) ] ;
- }
-
- rule g ( args * )
- {
- if $(x) in $(y)
- {
- return $(x) ;
- }
- else if $(y) in $(x)
- {
- return $(y) ;
- }
- else
- {
- return ;
- }
- }
-
- rule get-class ( )
- {
- return $(__class__) ;
- }
-
- rule get-instance ( )
- {
- return $(__name__) ;
- }
-
- rule invariant ( )
- {
- assert.equal 1 : 1 ;
- }
-
- rule get-foo ( )
- {
- return $(foo) ;
- }
- } # class myclass ;
-
- class derived1 : myclass
- {
- rule __init__ ( z_ )
- {
- myclass.__init__ $(z_) : X ;
- z = $(z_) ;
- }
-
- # Override g.
- #
- rule g ( args * )
- {
- return derived1.g ;
- }
-
- rule h ( )
- {
- return derived1.h ;
- }
-
- rule get-z ( )
- {
- return $(z) ;
- }
-
- # Check that 'assert.equal' visible in base class is visible here.
- #
- rule invariant2 ( )
- {
- assert.equal 2 : 2 ;
- }
-
- # Check that 'assert.variable-not-empty' visible in base class is
- # visible here.
- #
- rule invariant3 ( )
- {
- local v = 10 ;
- assert.variable-not-empty v ;
- }
- } # class derived1 : myclass ;
-
- class derived2 : myclass
- {
- rule __init__ ( )
- {
- myclass.__init__ 1 : 2 ;
- }
-
- # Override g.
- #
- rule g ( args * )
- {
- return derived2.g ;
- }
-
- # Test the ability to call base class functions with qualification.
- #
- rule get-x ( )
- {
- return [ myclass.get-x ] ;
- }
- } # class derived2 : myclass ;
-
- class derived2a : derived2
- {
- rule __init__
- {
- derived2.__init__ ;
- }
- } # class derived2a : derived2 ;
-
- local rule expect_derived2 ( [derived2] x ) { }
-
- local a = [ new myclass 3 4 5 : 4 5 ] ;
- local b = [ new derived1 4 ] ;
- local b2 = [ new derived1 4 ] ;
- local c = [ new derived2 ] ;
- local d = [ new derived2 ] ;
- local e = [ new derived2a ] ;
-
- expect_derived2 $(d) ;
- expect_derived2 $(e) ;
-
- # Argument checking is set up to call exit(1) directly on failure, and we
- # can not hijack that with try, so we should better not do this test by
- # default. We could fix this by having errors look up and invoke the EXIT
- # rule instead; EXIT can be hijacked (;-)
- if --fail-typecheck in [ modules.peek : ARGV ]
- {
- try ;
- {
- expect_derived2 $(a) ;
- }
- catch
- "Expected an instance of derived2 but got" instead
- ;
- }
-
- #try ;
- #{
- # new bad_subclass ;
- #}
- #catch
- # bad_subclass.bad_subclass failed to call base class constructor
- # myclass.__init__
- # ;
-
- #try ;
- #{
- # class bad_subclass ;
- #}
- #catch bad_subclass has already been declared ;
-
- assert.result 3 4 5 : $(a).get-x ;
- assert.result 4 5 : $(a).get-y ;
- assert.result 4 : $(b).get-x ;
- assert.result X : $(b).get-y ;
- assert.result 4 : $(b).get-z ;
- assert.result 1 : $(c).get-x ;
- assert.result 2 : $(c).get-y ;
- assert.result 4 5 : $(a).f ;
- assert.result derived1.g : $(b).f ;
- assert.result derived2.g : $(c).f ;
- assert.result derived2.g : $(d).f ;
-
- assert.result 10 : $(b).get-foo ;
-
- $(a).invariant ;
- $(b).invariant2 ;
- $(b).invariant3 ;
-
- # Check that the __class__ attribute is getting properly set.
- assert.result myclass : $(a).get-class ;
- assert.result derived1 : $(b).get-class ;
- assert.result $(a) : $(a).get-instance ;
-
- $(a).set-x a.x ;
- $(b).set-x b.x ;
- $(c).set-x c.x ;
- $(d).set-x d.x ;
- assert.result a.x : $(a).get-x ;
- assert.result b.x : $(b).get-x ;
- assert.result c.x : $(c).get-x ;
- assert.result d.x : $(d).get-x ;
-
- class derived3 : derived1 derived2
- {
- rule __init__ ( )
- {
- }
- }
-
- assert.result : bases myclass ;
- assert.result myclass : bases derived1 ;
- assert.result myclass : bases derived2 ;
- assert.result derived1 derived2 : bases derived3 ;
-
- assert.true is-derived derived1 : myclass ;
- assert.true is-derived derived2 : myclass ;
- assert.true is-derived derived3 : derived1 ;
- assert.true is-derived derived3 : derived2 ;
- assert.true is-derived derived3 : derived1 derived2 myclass ;
- assert.true is-derived derived3 : myclass ;
-
- assert.false is-derived myclass : derived1 ;
-
- assert.true is-instance $(a) ;
- assert.false is-instance bar ;
-
- assert.true is-a $(a) : myclass ;
- assert.true is-a $(c) : derived2 ;
- assert.true is-a $(d) : myclass ;
- assert.false is-a literal : myclass ;
-}
[46/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/build/feature.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/build/feature.jam b/ext/kenlm/jam-files/boost-build/build/feature.jam
deleted file mode 100644
index ee6abc5..0000000
--- a/ext/kenlm/jam-files/boost-build/build/feature.jam
+++ /dev/null
@@ -1,1350 +0,0 @@
-# Copyright 2001, 2002, 2003 Dave Abrahams
-# Copyright 2002, 2006 Rene Rivera
-# Copyright 2002, 2003, 2004, 2005, 2006 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-import assert : * ;
-import "class" : * ;
-import indirect ;
-import modules ;
-import regex ;
-import sequence ;
-import set ;
-import utility ;
-
-
-local rule setup ( )
-{
- .all-attributes =
- implicit
- composite
- optional
- symmetric
- free
- incidental
- path
- dependency
- propagated
- link-incompatible
- subfeature
- order-sensitive
- ;
-
- .all-features = ;
- .all-subfeatures = ;
- .all-top-features = ; # non-subfeatures
- .all-implicit-values = ;
-}
-setup ;
-
-
-# Prepare a fresh space to test in by moving all global variable settings into
-# the given temporary module and erasing them here.
-#
-rule prepare-test ( temp-module )
-{
- DELETE_MODULE $(temp-module) ;
-
- # Transfer globals to temp-module.
- for local v in [ VARNAMES feature ]
- {
- if [ MATCH (\\.) : $(v) ]
- {
- modules.poke $(temp-module) : $(v) : $($(v)) ;
- $(v) = ;
- }
- }
- setup ;
-}
-
-
-# Clear out all global variables and recover all variables from the given
-# temporary module.
-#
-rule finish-test ( temp-module )
-{
- # Clear globals.
- for local v in [ VARNAMES feature ]
- {
- if [ MATCH (\\.) : $(v) ]
- {
- $(v) = ;
- }
- }
-
- for local v in [ VARNAMES $(temp-module) ]
- {
- $(v) = [ modules.peek $(temp-module) : $(v) ] ;
- }
- DELETE_MODULE $(temp-module) ;
-}
-
-
-# Transform features by bracketing any elements which are not already bracketed
-# by "<>".
-#
-local rule grist ( features * )
-{
- local empty = "" ;
- return $(empty:G=$(features)) ;
-}
-
-
-# Declare a new feature with the given name, values, and attributes.
-#
-rule feature (
- name # Feature name.
- : values * # Allowable values - may be extended later using feature.extend.
- : attributes * # Feature attributes (e.g. implicit, free, propagated...).
-)
-{
- name = [ grist $(name) ] ;
-
- local error ;
-
- # Check for any unknown attributes.
- if ! ( $(attributes) in $(.all-attributes) )
- {
- error = unknown attributes:
- [ set.difference $(attributes) : $(.all-attributes) ] ;
- }
- else if $(name) in $(.all-features)
- {
- error = feature already defined: ;
- }
- else if implicit in $(attributes) && free in $(attributes)
- {
- error = free features cannot also be implicit ;
- }
- else if free in $(attributes) && propagated in $(attributes)
- {
- error = free features cannot be propagated ;
- }
- else
- {
- local m = [ MATCH (.*=.*) : $(values) ] ;
- if $(m[1])
- {
- error = "feature value may not contain '='" ;
- }
- }
-
- if $(error)
- {
- import errors ;
- errors.error $(error)
- : "in" feature declaration:
- : feature [ errors.lol->list $(1) : $(2) : $(3) ] ;
- }
-
- $(name).values ?= ;
- $(name).attributes = $(attributes) ;
- $(name).subfeatures ?= ;
- $(attributes).features += $(name) ;
-
- .all-features += $(name) ;
- if subfeature in $(attributes)
- {
- .all-subfeatures += $(name) ;
- }
- else
- {
- .all-top-features += $(name) ;
- }
- extend $(name) : $(values) ;
-}
-
-
-# Sets the default value of the given feature, overriding any previous default.
-#
-rule set-default ( feature : value )
-{
- local f = [ grist $(feature) ] ;
- local a = $($(f).attributes) ;
- local bad-attribute = ;
- if free in $(a)
- {
- bad-attribute = free ;
- }
- else if optional in $(a)
- {
- bad-attribute = optional ;
- }
- if $(bad-attribute)
- {
- import errors ;
- errors.error $(bad-attribute) property $(f) cannot have a default. ;
- }
- if ! $(value) in $($(f).values)
- {
- import errors ;
- errors.error The specified default value, '$(value)' is invalid :
- allowed values are: $($(f).values) ;
- }
- $(f).default = $(value) ;
-}
-
-
-# Returns the default property values for the given features.
-#
-rule defaults ( features * )
-{
- local result ;
- for local f in $(features)
- {
- local gf = $(:E=:G=$(f)) ;
- local a = $($(gf).attributes) ;
- if ( free in $(a) ) || ( optional in $(a) )
- {
- }
- else
- {
- result += $(gf)$($(gf).default) ;
- }
- }
- return $(result) ;
-}
-
-
-# Returns true iff all 'names' elements are valid features.
-#
-rule valid ( names + )
-{
- if $(names) in $(.all-features)
- {
- return true ;
- }
-}
-
-
-# Returns the attibutes of the given feature.
-#
-rule attributes ( feature )
-{
- return $($(feature).attributes) ;
-}
-
-
-# Returns the values of the given feature.
-#
-rule values ( feature )
-{
- return $($(:E=:G=$(feature)).values) ;
-}
-
-
-# Returns true iff 'value-string' is a value-string of an implicit feature.
-#
-rule is-implicit-value ( value-string )
-{
- local v = [ regex.split $(value-string) - ] ;
- local failed ;
- if ! $(v[1]) in $(.all-implicit-values)
- {
- failed = true ;
- }
- else
- {
- local feature = $($(v[1]).implicit-feature) ;
- for local subvalue in $(v[2-])
- {
- if ! [ find-implied-subfeature $(feature) $(subvalue) : $(v[1]) ]
- {
- failed = true ;
- }
- }
- }
-
- if ! $(failed)
- {
- return true ;
- }
-}
-
-
-# Returns the implicit feature associated with the given implicit value.
-#
-rule implied-feature ( implicit-value )
-{
- local components = [ regex.split $(implicit-value) "-" ] ;
- local feature = $($(components[1]).implicit-feature) ;
- if ! $(feature)
- {
- import errors ;
- errors.error \"$(implicit-value)\" is not an implicit feature value ;
- feature = "" ; # Keep testing happy; it expects a result.
- }
- return $(feature) ;
-}
-
-
-local rule find-implied-subfeature ( feature subvalue : value-string ? )
-{
- # Feature should be of the form <feature-name>.
- if $(feature) != $(feature:G)
- {
- import errors ;
- errors.error invalid feature $(feature) ;
- }
- return $($(feature)$(value-string:E="")<>$(subvalue).subfeature) ;
-}
-
-
-# Given a feature and a value of one of its subfeatures, find the name of the
-# subfeature. If value-string is supplied, looks for implied subfeatures that
-# are specific to that value of feature
-#
-rule implied-subfeature (
- feature # The main feature name.
- subvalue # The value of one of its subfeatures.
- : value-string ? # The value of the main feature.
-)
-{
- local subfeature = [ find-implied-subfeature $(feature) $(subvalue)
- : $(value-string) ] ;
- if ! $(subfeature)
- {
- value-string ?= "" ;
- import errors ;
- errors.error \"$(subvalue)\" is not a known subfeature value of
- $(feature)$(value-string) ;
- }
- return $(subfeature) ;
-}
-
-
-# Generate an error if the feature is unknown.
-#
-local rule validate-feature ( feature )
-{
- if ! $(feature) in $(.all-features)
- {
- import errors ;
- errors.error unknown feature \"$(feature)\" ;
- }
-}
-
-
-# Given a feature and its value or just a value corresponding to an implicit
-# feature, returns a property set consisting of all component subfeatures and
-# their values. For example all the following calls:
-#
-# expand-subfeatures-aux <toolset>gcc-2.95.2-linux-x86
-# expand-subfeatures-aux gcc-2.95.2-linux-x86
-#
-# return:
-#
-# <toolset>gcc <toolset-version>2.95.2 <toolset-os>linux <toolset-cpu>x86
-#
-local rule expand-subfeatures-aux (
- feature ? # Feature name or empty if value corresponds to an
- # implicit property.
- : value # Feature value.
- : dont-validate ? # If set, no value string validation will be done.
-)
-{
- if $(feature)
- {
- feature = $(feature) ;
- }
-
- if ! $(feature)
- {
- feature = [ implied-feature $(value) ] ;
- }
- else
- {
- validate-feature $(feature) ;
- }
- if ! $(dont-validate)
- {
- validate-value-string $(feature) $(value) ;
- }
-
- local components = [ regex.split $(value) "-" ] ;
-
- # Get the top-level feature's value.
- local value = $(components[1]:G=) ;
-
- local result = $(components[1]:G=$(feature)) ;
-
- local subvalues = $(components[2-]) ;
- while $(subvalues)
- {
- local subvalue = $(subvalues[1]) ; # Pop the head off of subvalues.
- subvalues = $(subvalues[2-]) ;
-
- local subfeature = [ find-implied-subfeature $(feature) $(subvalue) :
- $(value) ] ;
-
- # If no subfeature was found reconstitute the value string and use that.
- if ! $(subfeature)
- {
- result = $(components:J=-) ;
- result = $(result:G=$(feature)) ;
- subvalues = ; # Stop looping.
- }
- else
- {
- local f = [ MATCH ^<(.*)>$ : $(feature) ] ;
- result += $(subvalue:G=$(f)-$(subfeature)) ;
- }
- }
-
- return $(result) ;
-}
-
-
-# Make all elements of properties corresponding to implicit features explicit,
-# and express all subfeature values as separate properties in their own right.
-# For example, all of the following properties
-#
-# gcc-2.95.2-linux-x86
-# <toolset>gcc-2.95.2-linux-x86
-#
-# might expand to
-#
-# <toolset>gcc <toolset-version>2.95.2 <toolset-os>linux <toolset-cpu>x86
-#
-rule expand-subfeatures (
- properties * # Property set with elements of the form
- # <feature>value-string or just value-string in the case
- # of implicit features.
- : dont-validate ?
-)
-{
- local result ;
- for local p in $(properties)
- {
- # Don't expand subfeatures in subfeatures
- if ! [ MATCH "(:)" : $(p:G) ]
- {
- result += [ expand-subfeatures-aux $(p:G) : $(p:G=) : $(dont-validate) ] ;
- }
- else
- {
- result += $(p) ;
- }
- }
- return $(result) ;
-}
-
-
-# Helper for extend, below. Handles the feature case.
-#
-local rule extend-feature ( feature : values * )
-{
- feature = [ grist $(feature) ] ;
- validate-feature $(feature) ;
- if implicit in $($(feature).attributes)
- {
- for local v in $(values)
- {
- if $($(v).implicit-feature)
- {
- import errors ;
- errors.error $(v) is already associated with the
- \"$($(v).implicit-feature)\" feature ;
- }
- $(v).implicit-feature = $(feature) ;
- }
-
- .all-implicit-values += $(values) ;
- }
- if ! $($(feature).values)
- {
- # This is the first value specified for this feature so make it be the
- # default.
- $(feature).default = $(values[1]) ;
- }
- $(feature).values += $(values) ;
-}
-
-
-# Checks that value-string is a valid value-string for the given feature.
-#
-rule validate-value-string ( feature value-string )
-{
- if ! (
- free in $($(feature).attributes)
- || ( $(value-string) in $(feature).values )
- )
- {
- local values = $(value-string) ;
-
- if $($(feature).subfeatures)
- {
- if ! $(value-string) in $($(feature).values)
- $($(feature).subfeatures)
- {
- values = [ regex.split $(value-string) - ] ;
- }
- }
-
- if ! ( $(values[1]) in $($(feature).values) ) &&
-
- # An empty value is allowed for optional features.
- ( $(values[1]) || ! ( optional in $($(feature).attributes) ) )
- {
- import errors ;
- errors.error \"$(values[1])\" is not a known value of feature
- $(feature) : legal values: \"$($(feature).values)\" ;
- }
-
- for local v in $(values[2-])
- {
- # This will validate any subfeature values in value-string.
- implied-subfeature $(feature) $(v) : $(values[1]) ;
- }
- }
-}
-
-
-# A helper that computes:
-# * name(s) of module-local variable(s) used to record the correspondence
-# between subvalue(s) and a subfeature
-# * value of that variable when such a subfeature/subvalue has been defined and
-# returns a list consisting of the latter followed by the former.
-#
-local rule subvalue-var (
- feature # Main feature name.
- value-string ? # If supplied, specifies a specific value of the main
- # feature for which the subfeature values are valid.
- : subfeature # Subfeature name.
- : subvalues * # Subfeature values.
-)
-{
- feature = [ grist $(feature) ] ;
- validate-feature $(feature) ;
- if $(value-string)
- {
- validate-value-string $(feature) $(value-string) ;
- }
-
- local subfeature-name = [ get-subfeature-name $(subfeature) $(value-string) ] ;
-
- return $(subfeature-name)
- $(feature)$(value-string:E="")<>$(subvalues).subfeature ;
-}
-
-
-# Extends the given subfeature with the subvalues. If the optional value-string
-# is provided, the subvalues are only valid for the given value of the feature.
-# Thus, you could say that <target-platform>mingw is specific to
-# <toolset>gcc-2.95.2 as follows:
-#
-# extend-subfeature toolset gcc-2.95.2 : target-platform : mingw ;
-#
-rule extend-subfeature (
- feature # The feature whose subfeature is being extended.
-
- value-string ? # If supplied, specifies a specific value of the main
- # feature for which the new subfeature values are valid.
-
- : subfeature # Subfeature name.
- : subvalues * # Additional subfeature values.
-)
-{
- local subfeature-vars = [ subvalue-var $(feature) $(value-string)
- : $(subfeature) : $(subvalues) ] ;
-
- local f = [ utility.ungrist [ grist $(feature) ] ] ;
- extend $(f)-$(subfeature-vars[1]) : $(subvalues) ;
-
- # Provide a way to get from the given feature or property and subfeature
- # value to the subfeature name.
- $(subfeature-vars[2-]) = $(subfeature-vars[1]) ;
-}
-
-
-# Returns true iff the subvalues are valid for the feature. When the optional
-# value-string is provided, returns true iff the subvalues are valid for the
-# given value of the feature.
-#
-rule is-subvalue ( feature : value-string ? : subfeature : subvalue )
-{
- local subfeature-vars = [ subvalue-var $(feature) $(value-string)
- : $(subfeature) : $(subvalue) ] ;
-
- if $($(subfeature-vars[2])) = $(subfeature-vars[1])
- {
- return true ;
- }
-}
-
-
-# Can be called three ways:
-#
-# 1. extend feature : values *
-# 2. extend <feature> subfeature : values *
-# 3. extend <feature>value-string subfeature : values *
-#
-# * Form 1 adds the given values to the given feature.
-# * Forms 2 and 3 add subfeature values to the given feature.
-# * Form 3 adds the subfeature values as specific to the given property
-# value-string.
-#
-rule extend ( feature-or-property subfeature ? : values * )
-{
- local feature ; # If a property was specified this is its feature.
- local value-string ; # E.g., the gcc-2.95-2 part of <toolset>gcc-2.95.2.
-
- # If a property was specified.
- if $(feature-or-property:G) && $(feature-or-property:G=)
- {
- # Extract the feature and value-string, if any.
- feature = $(feature-or-property:G) ;
- value-string = $(feature-or-property:G=) ;
- }
- else
- {
- feature = [ grist $(feature-or-property) ] ;
- }
-
- # Dispatch to the appropriate handler.
- if $(subfeature)
- {
- extend-subfeature $(feature) $(value-string) : $(subfeature)
- : $(values) ;
- }
- else
- {
- # If no subfeature was specified, we do not expect to see a
- # value-string.
- if $(value-string)
- {
- import errors ;
- errors.error can only specify a property as the first argument when
- extending a subfeature
- : usage:
- : " extend" feature ":" values...
- : " | extend" <feature>value-string subfeature ":" values... ;
- }
-
- extend-feature $(feature) : $(values) ;
- }
-}
-
-
-local rule get-subfeature-name ( subfeature value-string ? )
-{
- local prefix = $(value-string): ;
- return $(prefix:E="")$(subfeature) ;
-}
-
-
-# Declares a subfeature.
-#
-rule subfeature (
- feature # Root feature that is not a subfeature.
- value-string ? # A value-string specifying which feature or subfeature
- # values this subfeature is specific to, if any.
- : subfeature # The name of the subfeature being declared.
- : subvalues * # The allowed values of this subfeature.
- : attributes * # The attributes of the subfeature.
-)
-{
- feature = [ grist $(feature) ] ;
- validate-feature $(feature) ;
-
- # Add grist to the subfeature name if a value-string was supplied.
- local subfeature-name = [ get-subfeature-name $(subfeature) $(value-string) ] ;
-
- if $(subfeature-name) in $($(feature).subfeatures)
- {
- import errors ;
- errors.error \"$(subfeature)\" already declared as a subfeature of
- \"$(feature)\" "specific to "$(value-string) ;
- }
- $(feature).subfeatures += $(subfeature-name) ;
-
- # First declare the subfeature as a feature in its own right.
- local f = [ utility.ungrist $(feature) ] ;
- feature $(f)-$(subfeature-name) : $(subvalues) : $(attributes) subfeature ;
-
- # Now make sure the subfeature values are known.
- extend-subfeature $(feature) $(value-string) : $(subfeature) : $(subvalues) ;
-}
-
-
-# Set components of the given composite property.
-#
-rule compose ( composite-property : component-properties * )
-{
- local feature = $(composite-property:G) ;
- if ! ( composite in [ attributes $(feature) ] )
- {
- import errors ;
- errors.error "$(feature)" is not a composite feature ;
- }
-
- $(composite-property).components ?= ;
- if $($(composite-property).components)
- {
- import errors ;
- errors.error components of "$(composite-property)" already set:
- $($(composite-property).components) ;
- }
-
- if $(composite-property) in $(component-properties)
- {
- import errors ;
- errors.error composite property "$(composite-property)" cannot have itself as a component ;
- }
- $(composite-property).components = $(component-properties) ;
-}
-
-
-local rule expand-composite ( property )
-{
- return $(property)
- [ sequence.transform expand-composite : $($(property).components) ] ;
-}
-
-
-# Return all values of the given feature specified by the given property set.
-#
-rule get-values ( feature : properties * )
-{
- local result ;
-
- feature = $(:E=:G=$(feature)) ; # Add <> if necessary.
- for local p in $(properties)
- {
- if $(p:G) = $(feature)
- {
- # Use MATCH instead of :G= to get the value, in order to preserve
- # the value intact instead of having bjam treat it as a decomposable
- # path.
- result += [ MATCH ">(.*)" : $(p) ] ;
- }
- }
- return $(result) ;
-}
-
-
-rule free-features ( )
-{
- return $(free.features) ;
-}
-
-
-# Expand all composite properties in the set so that all components are
-# explicitly expressed.
-#
-rule expand-composites ( properties * )
-{
- local explicit-features = $(properties:G) ;
- local result ;
-
- # Now expand composite features.
- for local p in $(properties)
- {
- local expanded = [ expand-composite $(p) ] ;
-
- for local x in $(expanded)
- {
- if ! $(x) in $(result)
- {
- local f = $(x:G) ;
-
- if $(f) in $(free.features)
- {
- result += $(x) ;
- }
- else if ! $(x) in $(properties) # x is the result of expansion
- {
- if ! $(f) in $(explicit-features) # not explicitly-specified
- {
- if $(f) in $(result:G)
- {
- import errors ;
- errors.error expansions of composite features result
- in conflicting values for $(f)
- : values: [ get-values $(f) : $(result) ] $(x:G=)
- : one contributing composite property was $(p) ;
- }
- else
- {
- result += $(x) ;
- }
- }
- }
- else if $(f) in $(result:G)
- {
- import errors ;
- errors.error explicitly-specified values of non-free feature
- $(f) conflict :
- "existing values:" [ get-values $(f) : $(properties) ] :
- "value from expanding " $(p) ":" $(x:G=) ;
- }
- else
- {
- result += $(x) ;
- }
- }
- }
- }
- return $(result) ;
-}
-
-
-# Return true iff f is an ordinary subfeature of the parent-property's feature,
-# or if f is a subfeature of the parent-property's feature specific to the
-# parent-property's value.
-#
-local rule is-subfeature-of ( parent-property f )
-{
- if subfeature in $($(f).attributes)
- {
- local specific-subfeature = [ MATCH <(.*):(.*)> : $(f) ] ;
- if $(specific-subfeature)
- {
- # The feature has the form <topfeature-topvalue:subfeature>, e.g.
- # <toolset-msvc:version>.
- local feature-value = [ split-top-feature $(specific-subfeature[1])
- ] ;
- if <$(feature-value[1])>$(feature-value[2]) = $(parent-property)
- {
- return true ;
- }
- }
- else
- {
- # The feature has the form <topfeature-subfeature>, e.g.
- # <toolset-version>
- local top-sub = [ split-top-feature [ utility.ungrist $(f) ] ] ;
- if $(top-sub[2]) && <$(top-sub[1])> = $(parent-property:G)
- {
- return true ;
- }
- }
- }
-}
-
-
-# As for is-subfeature-of but for subproperties.
-#
-local rule is-subproperty-of ( parent-property p )
-{
- return [ is-subfeature-of $(parent-property) $(p:G) ] ;
-}
-
-
-# Given a property, return the subset of features consisting of all ordinary
-# subfeatures of the property's feature, and all specific subfeatures of the
-# property's feature which are conditional on the property's value.
-#
-local rule select-subfeatures ( parent-property : features * )
-{
- return [ sequence.filter is-subfeature-of $(parent-property) : $(features) ] ;
-}
-
-
-# As for select-subfeatures but for subproperties.
-#
-local rule select-subproperties ( parent-property : properties * )
-{
- return [ sequence.filter is-subproperty-of $(parent-property) : $(properties) ] ;
-}
-
-
-# Given a property set which may consist of composite and implicit properties
-# and combined subfeature values, returns an expanded, normalized property set
-# with all implicit features expressed explicitly, all subfeature values
-# individually expressed, and all components of composite properties expanded.
-# Non-free features directly expressed in the input properties cause any values
-# of those features due to composite feature expansion to be dropped. If two
-# values of a given non-free feature are directly expressed in the input, an
-# error is issued.
-#
-rule expand ( properties * )
-{
- local expanded = [ expand-subfeatures $(properties) ] ;
- return [ expand-composites $(expanded) ] ;
-}
-
-
-# Helper rule for minimize. Returns true iff property's feature is present in
-# the contents of the variable named by feature-set-var.
-#
-local rule in-features ( feature-set-var property )
-{
- if $(property:G) in $($(feature-set-var))
- {
- return true ;
- }
-}
-
-
-# Helper rule for minimize. Returns the list with the same properties, but with
-# all subfeatures moved to the end of the list.
-#
-local rule move-subfeatures-to-the-end ( properties * )
-{
- local x1 ;
- local x2 ;
- for local p in $(properties)
- {
- if subfeature in $($(p:G).attributes)
- {
- x2 += $(p) ;
- }
- else
- {
- x1 += $(p) ;
- }
- }
- return $(x1) $(x2) ;
-}
-
-
-# Given an expanded property set, eliminate all redundancy: properties that are
-# elements of other (composite) properties in the set will be eliminated.
-# Non-symmetric properties equal to default values will be eliminated unless
-# they override a value from some composite property. Implicit properties will
-# be expressed without feature grist, and sub-property values will be expressed
-# as elements joined to the corresponding main property.
-#
-rule minimize ( properties * )
-{
- # Precondition checking
- local implicits = [ set.intersection $(p:G=) : $(p:G) ] ;
- if $(implicits)
- {
- import errors ;
- errors.error minimize requires an expanded property set, but
- \"$(implicits[1])\" appears to be the value of an un-expanded
- implicit feature ;
- }
-
- # Remove properties implied by composite features.
- local components = $($(properties).components) ;
- local x = [ set.difference $(properties) : $(components) ] ;
-
- # Handle subfeatures and implicit features.
- x = [ move-subfeatures-to-the-end $(x) ] ;
- local result ;
- while $(x)
- {
- local p fullp = $(x[1]) ;
- local f = $(p:G) ;
- local v = $(p:G=) ;
-
- # Eliminate features in implicit properties.
- if implicit in [ attributes $(f) ]
- {
- p = $(v) ;
- }
-
- # Locate all subproperties of $(x[1]) in the property set.
- local subproperties = [ select-subproperties $(fullp) : $(x) ] ;
- if $(subproperties)
- {
- # Reconstitute the joined property name.
- local sorted = [ sequence.insertion-sort $(subproperties) ] ;
- result += $(p)-$(sorted:G="":J=-) ;
-
- x = [ set.difference $(x[2-]) : $(subproperties) ] ;
- }
- else
- {
- # Eliminate properties whose value is equal to feature's default,
- # which are not symmetric and which do not contradict values implied
- # by composite properties.
-
- # Since all component properties of composites in the set have been
- # eliminated, any remaining property whose feature is the same as a
- # component of a composite in the set must have a non-redundant
- # value.
- if $(fullp) != [ defaults $(f) ]
- || symmetric in [ attributes $(f) ]
- || $(fullp:G) in $(components:G)
- {
- result += $(p) ;
- }
-
- x = $(x[2-]) ;
- }
- }
- return $(result) ;
-}
-
-
-# Combine all subproperties into their parent properties
-#
-# Requires: for every subproperty, there is a parent property. All features are
-# explicitly expressed.
-#
-# This rule probably should not be needed, but build-request.expand-no-defaults
-# is being abused for unintended purposes and it needs help.
-#
-rule compress-subproperties ( properties * )
-{
- local all-subs ;
- local matched-subs ;
- local result ;
-
- for local p in $(properties)
- {
- if ! $(p:G)
- {
- # Expecting fully-gristed properties.
- assert.variable-not-empty p:G ;
- }
-
- if ! subfeature in $($(p:G).attributes)
- {
- local subs = [ sequence.insertion-sort
- [ sequence.filter is-subproperty-of $(p) : $(properties) ] ] ;
-
- matched-subs += $(subs) ;
-
- local subvalues = -$(subs:G=:J=-) ;
- subvalues ?= "" ;
- result += $(p)$(subvalues) ;
- }
- else
- {
- all-subs += $(p) ;
- }
- }
- assert.result true : set.equal $(all-subs) : $(matched-subs) ;
- return $(result) ;
-}
-
-
-# Given an ungristed string, finds the longest prefix which is a top-level
-# feature name followed by a dash, and return a pair consisting of the parts
-# before and after that dash. More interesting than a simple split because
-# feature names may contain dashes.
-#
-local rule split-top-feature ( feature-plus )
-{
- local e = [ regex.split $(feature-plus) - ] ;
- local f = $(e[1]) ;
- local v ;
- while $(e)
- {
- if <$(f)> in $(.all-top-features)
- {
- v = $(f) $(e[2-]:J=-) ;
- }
- e = $(e[2-]) ;
- f = $(f)-$(e[1]) ;
- }
- return $(v) ;
-}
-
-
-# Given a set of properties, add default values for features not represented in
-# the set.
-#
-# Note: if there's an ordinary feature F1 and a composite feature F2 which
-# includes some value for F1 and both feature have default values then the
-# default value of F1 will be added (as opposed to the value in F2). This might
-# not be the right idea, e.g. consider:
-#
-# feature variant : debug ... ;
-# <variant>debug : .... <runtime-debugging>on
-# feature <runtime-debugging> : off on ;
-#
-# Here, when adding default for an empty property set, we'll get
-#
-# <variant>debug <runtime_debugging>off
-#
-# and that's kind of strange.
-#
-rule add-defaults ( properties * )
-{
- for local v in $(properties:G=)
- {
- if $(v) in $(properties)
- {
- import errors ;
- errors.error add-defaults requires explicitly specified features,
- but \"$(v)\" appears to be the value of an un-expanded implicit
- feature ;
- }
- }
- # We don't add default for elements with ":" inside. This catches:
- # 1. Conditional properties --- we don't want <variant>debug:<define>DEBUG
- # to be takes as specified value for <variant>
- # 2. Free properties with ":" in values. We don't care, since free
- # properties don't have defaults.
- local xproperties = [ MATCH "^([^:]+)$" : $(properties) ] ;
- local missing-top = [ set.difference $(.all-top-features) : $(xproperties:G) ] ;
- local more = [ defaults $(missing-top) ] ;
- properties += $(more) ;
- xproperties += $(more) ;
-
- # Add defaults for subfeatures of features which are present.
- for local p in $(xproperties)
- {
- local s = $($(p:G).subfeatures) ;
- local f = [ utility.ungrist $(p:G) ] ;
- local missing-subs = [ set.difference <$(f)-$(s)> : $(properties:G) ] ;
- properties += [ defaults [ select-subfeatures $(p) : $(missing-subs) ] ] ;
- }
-
- return $(properties) ;
-}
-
-
-# Given a property-set of the form
-# v1/v2/...vN-1/<fN>vN/<fN+1>vN+1/...<fM>vM
-#
-# Returns
-# v1 v2 ... vN-1 <fN>vN <fN+1>vN+1 ... <fM>vM
-#
-# Note that vN...vM may contain slashes. This needs to be resilient to the
-# substitution of backslashes for slashes, since Jam, unbidden, sometimes swaps
-# slash direction on NT.
-#
-rule split ( property-set )
-{
- local pieces = [ regex.split $(property-set) [\\/] ] ;
- local result ;
-
- for local x in $(pieces)
- {
- if ( ! $(x:G) ) && $(result[-1]:G)
- {
- result = $(result[1--2]) $(result[-1])/$(x) ;
- }
- else
- {
- result += $(x) ;
- }
- }
-
- return $(result) ;
-}
-
-
-# Tests of module feature.
-#
-rule __test__ ( )
-{
- # Use a fresh copy of the feature module.
- prepare-test feature-test-temp ;
-
- import assert ;
- import errors : try catch ;
-
- # These are local rules and so must be explicitly reimported into the
- # testing module.
- import feature : extend-feature validate-feature select-subfeatures ;
-
- feature toolset : gcc : implicit ;
- feature define : : free ;
- feature runtime-link : dynamic static : symmetric ;
- feature optimization : on off ;
- feature variant : debug release profile : implicit composite symmetric ;
- feature stdlib : native stlport ;
- feature magic : : free ;
-
- compose <variant>debug : <define>_DEBUG <optimization>off ;
- compose <variant>release : <define>NDEBUG <optimization>on ;
-
- assert.result dynamic static : values <runtime-link> ;
- assert.result dynamic static : values runtime-link ;
-
- try ;
- {
- compose <variant>profile : <variant>profile ;
- }
- catch composite property <variant>profile cannot have itself as a component ;
-
- extend-feature toolset : msvc metrowerks ;
- subfeature toolset gcc : version : 2.95.2 2.95.3 2.95.4 3.0 3.0.1 3.0.2 ;
-
- assert.true is-subvalue toolset : gcc : version : 2.95.3 ;
- assert.false is-subvalue toolset : gcc : version : 1.1 ;
-
- assert.false is-subvalue toolset : msvc : version : 2.95.3 ;
- assert.false is-subvalue toolset : : version : yabba ;
-
- feature yabba ;
- subfeature yabba : version : dabba ;
- assert.true is-subvalue yabba : : version : dabba ;
-
- subfeature toolset gcc : platform : linux cygwin : optional ;
-
- assert.result <toolset-gcc:version>
- : select-subfeatures <toolset>gcc
- : <toolset-gcc:version>
- <toolset-msvc:version>
- <toolset-version>
- <stdlib> ;
-
- subfeature stdlib : version : 3 4 : optional ;
-
- assert.result <stdlib-version>
- : select-subfeatures <stdlib>native
- : <toolset-gcc:version>
- <toolset-msvc:version>
- <toolset-version>
- <stdlib-version> ;
-
- assert.result <toolset>gcc <toolset-gcc:version>3.0.1
- : expand-subfeatures <toolset>gcc-3.0.1 ;
-
- assert.result <toolset>gcc <toolset-gcc:version>3.0.1 <toolset-gcc:platform>linux
- : expand-subfeatures <toolset>gcc-3.0.1-linux ;
-
- assert.result <toolset>gcc <toolset-gcc:version>3.0.1
- : expand <toolset>gcc <toolset-gcc:version>3.0.1 ;
-
- assert.result <define>foo=x-y
- : expand-subfeatures <define>foo=x-y ;
-
- assert.result <toolset>gcc <toolset-gcc:version>3.0.1
- : expand-subfeatures gcc-3.0.1 ;
-
- assert.result a c e
- : get-values <x> : <x>a <y>b <x>c <y>d <x>e ;
-
- assert.result <toolset>gcc <toolset-gcc:version>3.0.1
- <variant>debug <define>_DEBUG <optimization>on
- : expand gcc-3.0.1 debug <optimization>on ;
-
- assert.result <variant>debug <define>_DEBUG <optimization>on
- : expand debug <optimization>on ;
-
- assert.result <optimization>on <variant>debug <define>_DEBUG
- : expand <optimization>on debug ;
-
- assert.result <runtime-link>dynamic <optimization>on
- : defaults <runtime-link> <define> <optimization> ;
-
- # Make sure defaults is resilient to missing grist.
- assert.result <runtime-link>dynamic <optimization>on
- : defaults runtime-link define optimization ;
-
- feature dummy : dummy1 dummy2 ;
- subfeature dummy : subdummy : x y z : optional ;
-
- feature fu : fu1 fu2 : optional ;
- subfeature fu : subfu : x y z : optional ;
- subfeature fu : subfu2 : q r s ;
-
- assert.result optional : attributes <fu> ;
-
- assert.result <runtime-link>static <define>foobar <optimization>on
- <toolset>gcc:<define>FOO <toolset>gcc <variant>debug <stdlib>native
- <dummy>dummy1 <toolset-gcc:version>2.95.2
- : add-defaults <runtime-link>static <define>foobar <optimization>on
- <toolset>gcc:<define>FOO ;
-
- assert.result <runtime-link>static <define>foobar <optimization>on
- <toolset>gcc:<define>FOO <fu>fu1 <toolset>gcc <variant>debug
- <stdlib>native <dummy>dummy1 <fu-subfu2>q <toolset-gcc:version>2.95.2
- : add-defaults <runtime-link>static <define>foobar <optimization>on
- <toolset>gcc:<define>FOO <fu>fu1 ;
-
- set-default <runtime-link> : static ;
- assert.result <runtime-link>static : defaults <runtime-link> ;
-
- assert.result gcc-3.0.1 debug <optimization>on
- : minimize [ expand gcc-3.0.1 debug <optimization>on <stdlib>native ] ;
-
- assert.result gcc-3.0.1 debug <runtime-link>dynamic
- : minimize
- [ expand gcc-3.0.1 debug <optimization>off <runtime-link>dynamic ] ;
-
- assert.result gcc-3.0.1 debug
- : minimize [ expand gcc-3.0.1 debug <optimization>off ] ;
-
- assert.result debug <optimization>on
- : minimize [ expand debug <optimization>on ] ;
-
- assert.result gcc-3.0
- : minimize <toolset>gcc <toolset-gcc:version>3.0 ;
-
- assert.result gcc-3.0
- : minimize <toolset-gcc:version>3.0 <toolset>gcc ;
-
- assert.result <x>y/z <a>b/c <d>e/f
- : split <x>y/z/<a>b/c/<d>e/f ;
-
- assert.result <x>y/z <a>b/c <d>e/f
- : split <x>y\\z\\<a>b\\c\\<d>e\\f ;
-
- assert.result a b c <d>e/f/g <h>i/j/k
- : split a/b/c/<d>e/f/g/<h>i/j/k ;
-
- assert.result a b c <d>e/f/g <h>i/j/k
- : split a\\b\\c\\<d>e\\f\\g\\<h>i\\j\\k ;
-
- # Test error checking.
-
- try ;
- {
- expand release <optimization>off <optimization>on ;
- }
- catch explicitly-specified values of non-free feature <optimization> conflict ;
-
- try ;
- {
- validate-feature <foobar> ;
- }
- catch unknown feature ;
-
- validate-value-string <toolset> gcc ;
- validate-value-string <toolset> gcc-3.0.1 ;
-
- try ;
- {
- validate-value-string <toolset> digital_mars ;
- }
- catch \"digital_mars\" is not a known value of <toolset> ;
-
- try ;
- {
- feature foobar : : baz ;
- }
- catch unknown attributes: baz ;
-
- feature feature1 ;
- try ;
- {
- feature feature1 ;
- }
- catch feature already defined: ;
-
- try ;
- {
- feature feature2 : : free implicit ;
- }
- catch free features cannot also be implicit ;
-
- try ;
- {
- feature feature3 : : free propagated ;
- }
- catch free features cannot be propagated ;
-
- try ;
- {
- implied-feature lackluster ;
- }
- catch \"lackluster\" is not an implicit feature value ;
-
- try ;
- {
- implied-subfeature <toolset> 3.0.1 ;
- }
- catch \"3.0.1\" is not a known subfeature value of <toolset> ;
-
- try ;
- {
- implied-subfeature <toolset> not-a-version : gcc ;
- }
- catch \"not-a-version\" is not a known subfeature value of <toolset>gcc ;
-
- # Leave a clean copy of the features module behind.
- finish-test feature-test-temp ;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/build/generators.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/build/generators.jam b/ext/kenlm/jam-files/boost-build/build/generators.jam
deleted file mode 100644
index ec7183a..0000000
--- a/ext/kenlm/jam-files/boost-build/build/generators.jam
+++ /dev/null
@@ -1,1420 +0,0 @@
-# Copyright 2002. Vladimir Prus
-# Copyright 2006. Rene Rivera
-#
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# Manages 'generators' --- objects which can do transformation between different
-# target types and contain algorithm for finding transformation from sources to
-# targets.
-#
-# The main entry point to this module is generators.construct rule. It is given
-# a list of source targets, desired target type and a set of properties. It
-# starts by selecting 'viable generators', which have any chances of producing
-# the desired target type with the required properties. Generators are ranked
-# and a set of the most specific ones is selected.
-#
-# The most specific generators have their 'run' methods called, with the
-# properties and list of sources. Each one selects a target which can be
-# directly consumed, and tries to convert the remaining ones to the types it can
-# consume. This is done by recursively calling 'construct' with all consumable
-# types.
-#
-# If the generator has collected all the targets it needs, it creates targets
-# corresponding to result, and returns it. When all generators have been run,
-# results of one of them are selected and returned as a result.
-#
-# It is quite possible for 'construct' to return more targets that it was asked
-# for. For example, if it were asked to generate a target of type EXE, but the
-# only found generator produces both EXE and TDS (file with debug) information.
-# The extra target will be returned.
-#
-# Likewise, when generator tries to convert sources to consumable types, it can
-# get more targets that it was asked for. The question is what to do with extra
-# targets. Boost.Build attempts to convert them to requested types, and attempts
-# that as early as possible. Specifically, this is done after invoking each
-# generator. TODO: An example is needed to document the rationale for trying
-# extra target conversion at that point.
-#
-# In order for the system to be able to use a specific generator instance 'when
-# needed', the instance needs to be registered with the system using
-# generators.register() or one of its related rules. Unregistered generators may
-# only be run explicitly and will not be considered by Boost.Build when when
-# converting between given target types.
-
-import "class" : new ;
-import property-set ;
-import sequence ;
-import set ;
-import type ;
-import utility ;
-import virtual-target ;
-
-
-if "--debug-generators" in [ modules.peek : ARGV ]
-{
- .debug = true ;
-}
-
-
-# Updated cached viable source target type information as needed after a new
-# target type gets defined. This is needed because if a target type is a viable
-# source target type for some generator then all of the target type's derived
-# target types should automatically be considered as viable source target types
-# for the same generator as well. Does nothing if a non-derived target type is
-# passed to it.
-#
-rule update-cached-information-with-a-new-type ( type )
-{
- local base-type = [ type.base $(type) ] ;
- if $(base-type)
- {
- for local g in $(.vstg-cached-generators)
- {
- if $(base-type) in $(.vstg.$(g))
- {
- .vstg.$(g) += $(type) ;
- }
- }
-
- for local t in $(.vst-cached-types)
- {
- if $(base-type) in $(.vst.$(t))
- {
- .vst.$(t) += $(type) ;
- }
- }
- }
-}
-
-
-# Clears cached viable source target type information except for target types
-# and generators with all source types listed as viable. Should be called when
-# something invalidates those cached values by possibly causing some new source
-# types to become viable.
-#
-local rule invalidate-extendable-viable-source-target-type-cache ( )
-{
- local generators-with-cached-source-types = $(.vstg-cached-generators) ;
- .vstg-cached-generators = ;
- for local g in $(generators-with-cached-source-types)
- {
- if $(.vstg.$(g)) = *
- {
- .vstg-cached-generators += $(g) ;
- }
- else
- {
- .vstg.$(g) = ;
- }
- }
-
- local types-with-cached-source-types = $(.vst-cached-types) ;
- .vst-cached-types = ;
- for local t in $(types-with-cached-source-types)
- {
- if $(.vst.$(t)) = *
- {
- .vst-cached-types += $(t) ;
- }
- else
- {
- .vst.$(t) = ;
- }
- }
-}
-
-
-# Outputs a debug message if generators debugging is on. Each element of
-# 'message' is checked to see if it is a class instance. If so, instead of the
-# value, the result of 'str' call is output.
-#
-local rule generators.dout ( message * )
-{
- if $(.debug)
- {
- ECHO [ sequence.transform utility.str : $(message) ] ;
- }
-}
-
-
-local rule indent ( )
-{
- return $(.indent:J="") ;
-}
-
-
-local rule increase-indent ( )
-{
- .indent += " " ;
-}
-
-
-local rule decrease-indent ( )
-{
- .indent = $(.indent[2-]) ;
-}
-
-
-# Models a generator.
-#
-class generator
-{
- import "class" : new ;
- import feature ;
- import generators : indent increase-indent decrease-indent generators.dout ;
- import utility ;
- import path ;
- import property ;
- import sequence ;
- import set ;
- import type ;
- import virtual-target ;
-
- EXPORT class@generator : indent increase-indent decrease-indent
- generators.dout ;
-
- rule __init__ (
- id # Identifies the generator - should be name
- # of the rule which sets up the build
- # actions.
-
- composing ? # Whether generator processes each source
- # target in turn, converting it to required
- # types. Ordinary generators pass all
- # sources together to the recursive
- # generators.construct-types call.
-
- : source-types * # Types that this generator can handle. If
- # empty, the generator can consume anything.
-
- : target-types-and-names + # Types the generator will create and,
- # optionally, names for created targets.
- # Each element should have the form
- # type["(" name-pattern ")"], for example,
- # obj(%_x). Generated target name will be
- # found by replacing % with the name of
- # source, provided an explicit name was not
- # specified.
-
- : requirements *
- )
- {
- self.id = $(id) ;
- self.rule-name = $(id) ;
- self.composing = $(composing) ;
- self.source-types = $(source-types) ;
- self.target-types-and-names = $(target-types-and-names) ;
- self.requirements = $(requirements) ;
-
- for local e in $(target-types-and-names)
- {
- # Create three parallel lists: one with the list of target types,
- # and two other with prefixes and postfixes to be added to target
- # name. We use parallel lists for prefix and postfix (as opposed to
- # mapping), because given target type might occur several times, for
- # example "H H(%_symbols)".
- local m = [ MATCH ([^\\(]*)(\\((.*)%(.*)\\))? : $(e) ] ;
- self.target-types += $(m[1]) ;
- self.name-prefix += $(m[3]:E="") ;
- self.name-postfix += $(m[4]:E="") ;
- }
-
- for local r in [ requirements ]
- {
- if $(r:G=)
- {
- self.property-requirements += $(r) ;
- }
- else
- {
- self.feature-requirements += $(r) ;
- }
- }
-
- # Note that 'transform' here, is the same as 'for_each'.
- sequence.transform type.validate : $(self.source-types) ;
- sequence.transform type.validate : $(self.target-types) ;
- }
-
- ################# End of constructor #################
-
- rule id ( )
- {
- return $(self.id) ;
- }
-
- # Returns the list of target type the generator accepts.
- #
- rule source-types ( )
- {
- return $(self.source-types) ;
- }
-
- # Returns the list of target types that this generator produces. It is
- # assumed to be always the same -- i.e. it can not change depending on some
- # provided list of sources.
- #
- rule target-types ( )
- {
- return $(self.target-types) ;
- }
-
- # Returns the required properties for this generator. Properties in returned
- # set must be present in build properties if this generator is to be used.
- # If result has grist-only element, that build properties must include some
- # value of that feature.
- #
- # XXX: remove this method?
- #
- rule requirements ( )
- {
- return $(self.requirements) ;
- }
-
- rule set-rule-name ( rule-name )
- {
- self.rule-name = $(rule-name) ;
- }
-
- rule rule-name ( )
- {
- return $(self.rule-name) ;
- }
-
- # Returns a true value if the generator can be run with the specified
- # properties.
- #
- rule match-rank ( property-set-to-match )
- {
- # See if generator requirements are satisfied by 'properties'. Treat a
- # feature name in requirements (i.e. grist-only element), as matching
- # any value of the feature.
-
- if [ $(property-set-to-match).contains-raw $(self.property-requirements) ] &&
- [ $(property-set-to-match).contains-features $(self.feature-requirements) ]
- {
- return true ;
- }
- else
- {
- return ;
- }
- }
-
- # Returns another generator which differs from $(self) in
- # - id
- # - value to <toolset> feature in properties
- #
- rule clone ( new-id : new-toolset-properties + )
- {
- local g = [ new $(__class__) $(new-id) $(self.composing) :
- $(self.source-types) : $(self.target-types-and-names) :
- # Note: this does not remove any subfeatures of <toolset> which
- # might cause problems.
- [ property.change $(self.requirements) : <toolset> ]
- $(new-toolset-properties) ] ;
- return $(g) ;
- }
-
- # Creates another generator that is the same as $(self), except that if
- # 'base' is in target types of $(self), 'type' will in target types of the
- # new generator.
- #
- rule clone-and-change-target-type ( base : type )
- {
- local target-types ;
- for local t in $(self.target-types-and-names)
- {
- local m = [ MATCH ([^\\(]*)(\\(.*\\))? : $(t) ] ;
- if $(m) = $(base)
- {
- target-types += $(type)$(m[2]:E="") ;
- }
- else
- {
- target-types += $(t) ;
- }
- }
-
- local g = [ new $(__class__) $(self.id) $(self.composing) :
- $(self.source-types) : $(target-types) : $(self.requirements) ] ;
- if $(self.rule-name)
- {
- $(g).set-rule-name $(self.rule-name) ;
- }
- return $(g) ;
- }
-
- # Tries to invoke this generator on the given sources. Returns a list of
- # generated targets (instances of 'virtual-target') and optionally a set of
- # properties to be added to the usage-requirements for all the generated
- # targets. Returning nothing from run indicates that the generator was
- # unable to create the target.
- #
- rule run
- (
- project # Project for which the targets are generated.
- name ? # Used when determining the 'name' attribute for all
- # generated targets. See the 'generated-targets' method.
- : property-set # Desired properties for generated targets.
- : sources + # Source targets.
- )
- {
- generators.dout [ indent ] " ** generator" $(self.id) ;
- generators.dout [ indent ] " composing:" $(self.composing) ;
-
- if ! $(self.composing) && $(sources[2]) && $(self.source-types[2])
- {
- import errors : error : errors.error ;
- errors.error "Unsupported source/source-type combination" ;
- }
-
- # We do not run composing generators if no name is specified. The reason
- # is that composing generator combines several targets, which can have
- # different names, and it cannot decide which name to give for produced
- # target. Therefore, the name must be passed.
- #
- # This in effect, means that composing generators are runnable only at
- # the top-level of a transformation graph, or if their name is passed
- # explicitly. Thus, we dissallow composing generators in the middle. For
- # example, the transformation CPP -> OBJ -> STATIC_LIB -> RSP -> EXE
- # will not be allowed as the OBJ -> STATIC_LIB generator is composing.
- if ! $(self.composing) || $(name)
- {
- run-really $(project) $(name) : $(property-set) : $(sources) ;
- }
- }
-
- rule run-really ( project name ? : property-set : sources + )
- {
- # Targets that this generator will consume directly.
- local consumed = ;
- # Targets that can not be consumed and will be returned as-is.
- local bypassed = ;
-
- if $(self.composing)
- {
- consumed = [ convert-multiple-sources-to-consumable-types $(project)
- : $(property-set) : $(sources) ] ;
- }
- else
- {
- consumed = [ convert-to-consumable-types $(project) $(name)
- : $(property-set) : $(sources) ] ;
- }
-
- local result ;
- if $(consumed)
- {
- result = [ construct-result $(consumed) : $(project) $(name) :
- $(property-set) ] ;
- }
-
- if $(result)
- {
- generators.dout [ indent ] " SUCCESS: " $(result) ;
- }
- else
- {
- generators.dout [ indent ] " FAILURE" ;
- }
- generators.dout ;
- return $(result) ;
- }
-
- # Constructs the dependency graph to be returned by this generator.
- #
- rule construct-result
- (
- consumed + # Already prepared list of consumable targets.
- # Composing generators may receive multiple sources
- # all of which will have types matching those in
- # $(self.source-types). Non-composing generators with
- # multiple $(self.source-types) will receive exactly
- # len $(self.source-types) sources with types matching
- # those in $(self.source-types). And non-composing
- # generators with only a single source type may
- # receive multiple sources with all of them of the
- # type listed in $(self.source-types).
- : project name ?
- : property-set # Properties to be used for all actions created here.
- )
- {
- local result ;
- # If this is a 1->1 transformation, apply it to all consumed targets in
- # order.
- if ! $(self.source-types[2]) && ! $(self.composing)
- {
- for local r in $(consumed)
- {
- result += [ generated-targets $(r) : $(property-set) :
- $(project) $(name) ] ;
- }
- }
- else if $(consumed)
- {
- result += [ generated-targets $(consumed) : $(property-set) :
- $(project) $(name) ] ;
- }
- return $(result) ;
- }
-
- # Determine target name from fullname (maybe including path components)
- # Place optional prefix and postfix around basename
- #
- rule determine-target-name ( fullname : prefix ? : postfix ? )
- {
- # See if we need to add directory to the target name.
- local dir = $(fullname:D) ;
- local name = $(fullname:B) ;
-
- name = $(prefix:E=)$(name) ;
- name = $(name)$(postfix:E=) ;
-
- if $(dir)
- # Never append '..' to target path.
- && ! [ MATCH .*(\\.\\.).* : $(dir) ]
- && ! [ path.is-rooted $(dir) ]
- {
- # Relative path is always relative to the source directory. Retain
- # it, so that users can have files with the same name in two
- # different subdirectories.
- name = $(dir)/$(name) ;
- }
- return $(name) ;
- }
-
- # Determine the name of the produced target from the names of the sources.
- #
- rule determine-output-name ( sources + )
- {
- # The simple case if when a name of source has single dot. Then, we take
- # the part before dot. Several dots can be caused by:
- # - using source file like a.host.cpp, or
- # - a type whose suffix has a dot. Say, we can type 'host_cpp' with
- # extension 'host.cpp'.
- # In the first case, we want to take the part up to the last dot. In the
- # second case -- not sure, but for now take the part up to the last dot
- # too.
- name = [ utility.basename [ $(sources[1]).name ] ] ;
- for local s in $(sources[2-])
- {
- if [ utility.basename [ $(s).name ] ] != $(name)
- {
- import errors : error : errors.error ;
- errors.error "$(self.id): source targets have different names: cannot determine target name" ;
- }
- }
- return [ determine-target-name [ $(sources[1]).name ] ] ;
- }
-
- # Constructs targets that are created after consuming 'sources'. The result
- # will be the list of virtual-target, which has the same length as the
- # 'target-types' attribute and with corresponding types.
- #
- # When 'name' is empty, all source targets must have the same 'name'
- # attribute value, which will be used instead of the 'name' argument.
- #
- # The 'name' attribute value for each generated target will be equal to the
- # 'name' parameter if there is no name pattern for this type. Otherwise, the
- # '%' symbol in the name pattern will be replaced with the 'name' parameter
- # to obtain the 'name' attribute.
- #
- # For example, if targets types are T1 and T2 (with name pattern "%_x"),
- # suffixes for T1 and T2 are .t1 and .t2, and source is foo.z, then created
- # files would be "foo.t1" and "foo_x.t2". The 'name' attribute actually
- # determines the basename of a file.
- #
- # Note that this pattern mechanism has nothing to do with implicit patterns
- # in make. It is a way to produce a target whose name is different than the
- # name of its source.
- #
- rule generated-targets ( sources + : property-set : project name ? )
- {
- if ! $(name)
- {
- name = [ determine-output-name $(sources) ] ;
- }
-
- # Assign an action for each target.
- local action = [ action-class ] ;
- local a = [ class.new $(action) $(sources) : $(self.rule-name) :
- $(property-set) ] ;
-
- # Create generated target for each target type.
- local targets ;
- local pre = $(self.name-prefix) ;
- local post = $(self.name-postfix) ;
- for local t in $(self.target-types)
- {
- local generated-name = $(pre[1])$(name:BS)$(post[1]) ;
- generated-name = $(generated-name:R=$(name:D)) ;
- pre = $(pre[2-]) ;
- post = $(post[2-]) ;
-
- targets += [ class.new file-target $(generated-name) : $(t) :
- $(project) : $(a) ] ;
- }
-
- return [ sequence.transform virtual-target.register : $(targets) ] ;
- }
-
- # Attempts to convert 'sources' to targets of types that this generator can
- # handle. The intention is to produce the set of targets that can be used
- # when the generator is run.
- #
- rule convert-to-consumable-types
- (
- project name ?
- : property-set
- : sources +
- : only-one ? # Convert 'source' to only one of the source types. If
- # there is more that one possibility, report an error.
- )
- {
- local _consumed ;
- local missing-types ;
-
- if $(sources[2])
- {
- # Do not know how to handle several sources yet. Just try to pass
- # the request to other generator.
- missing-types = $(self.source-types) ;
- }
- else
- {
- local temp = [ consume-directly $(sources) ] ;
- if $(temp[1])
- {
- _consumed = $(temp[1]) ;
- }
- missing-types = $(temp[2-]) ;
- }
-
- # No need to search for transformation if some source type has consumed
- # source and no more source types are needed.
- if $(only-one) && $(_consumed)
- {
- missing-types = ;
- }
-
- # TODO: we should check that only one source type is created if
- # 'only-one' is true.
-
- if $(missing-types)
- {
- local transformed = [ generators.construct-types $(project) $(name)
- : $(missing-types) : $(property-set) : $(sources) ] ;
-
- # Add targets of right type to 'consumed'. Add others to 'bypassed'.
- # The 'generators.construct' rule has done its best to convert
- # everything to the required type. There is no need to rerun it on
- # targets of different types.
-
- # NOTE: ignoring usage requirements.
- for local t in $(transformed[2-])
- {
- if [ $(t).type ] in $(missing-types)
- {
- _consumed += $(t) ;
- }
- }
- }
-
- return [ sequence.unique $(_consumed) ] ;
- }
-
- # Converts several files to consumable types. Called for composing
- # generators only.
- #
- rule convert-multiple-sources-to-consumable-types ( project : property-set :
- sources * )
- {
- local result ;
- # We process each source one-by-one, trying to convert it to a usable
- # type.
- if ! $(self.source-types)
- {
- # Anything is acceptible
- return $(sources) ;
- }
- else
- {
- local acceptible-types = [ sequence.unique
- [ sequence.transform type.all-derived : $(self.source-types) ] ] ;
- for local source in $(sources)
- {
- if ! [ $(source).type ] in $(acceptible-types)
- {
- local transformed = [ generators.construct-types $(project)
- : $(self.source-types) : $(property-set) : $(source) ] ;
- for local t in $(transformed[2-])
- {
- if [ $(t).type ] in $(self.source-types)
- {
- result += $(t) ;
- }
- }
- if ! $(transformed)
- {
- generators.dout [ indent ] " failed to convert " $(source) ;
- }
- }
- else
- {
- result += $(source) ;
- }
- }
- return [ sequence.unique $(result) ] ;
- }
- }
-
- rule consume-directly ( source )
- {
- local real-source-type = [ $(source).type ] ;
-
- # If there are no source types, we can consume anything.
- local source-types = $(self.source-types) ;
- source-types ?= $(real-source-type) ;
-
- local result = "" ;
- local missing-types ;
-
- for local st in $(source-types)
- {
- # The 'source' if of the right type already.
- if $(real-source-type) = $(st) || [ type.is-derived
- $(real-source-type) $(st) ]
- {
- result = $(source) ;
- }
- else
- {
- missing-types += $(st) ;
- }
- }
- return $(result) $(missing-types) ;
- }
-
- # Returns the class to be used to actions. Default implementation returns
- # "action".
- #
- rule action-class ( )
- {
- return "action" ;
- }
-}
-
-
-# Registers a new generator instance 'g'.
-#
-rule register ( g )
-{
- .all-generators += $(g) ;
-
- # A generator can produce several targets of the same type. We want unique
- # occurrence of that generator in .generators.$(t) in that case, otherwise,
- # it will be tried twice and we will get a false ambiguity.
- for local t in [ sequence.unique [ $(g).target-types ] ]
- {
- .generators.$(t) += $(g) ;
- }
-
- # Update the set of generators for toolset.
-
- # TODO: should we check that generator with this id is not already
- # registered. For example, the fop.jam module intentionally declared two
- # generators with the same id, so such check will break it.
- local id = [ $(g).id ] ;
-
- # Some generators have multiple periods in their name, so a simple $(id:S=)
- # will not generate the right toolset name. E.g. if id = gcc.compile.c++,
- # then .generators-for-toolset.$(id:S=) will append to
- # .generators-for-toolset.gcc.compile, which is a separate value from
- # .generators-for-toolset.gcc. Correcting this makes generator inheritance
- # work properly. See also inherit-generators in the toolset module.
- local base = $(id) ;
- while $(base:S)
- {
- base = $(base:B) ;
- }
- .generators-for-toolset.$(base) += $(g) ;
-
-
- # After adding a new generator that can construct new target types, we need
- # to clear the related cached viable source target type information for
- # constructing a specific target type or using a specific generator. Cached
- # viable source target type lists affected by this are those containing any
- # of the target types constructed by the new generator or any of their base
- # target types.
- #
- # A more advanced alternative to clearing that cached viable source target
- # type information would be to expand it with additional source types or
- # even better - mark it as needing to be expanded on next use.
- #
- # Also see the http://thread.gmane.org/gmane.comp.lib.boost.build/19077
- # mailing list thread for an even more advanced idea of how we could convert
- # Boost Build's Jamfile processing, target selection and generator selection
- # into separate steps which would prevent these caches from ever being
- # invalidated.
- #
- # For now we just clear all the cached viable source target type information
- # that does not simply state 'all types' and may implement a more detailed
- # algorithm later on if it becomes needed.
-
- invalidate-extendable-viable-source-target-type-cache ;
-}
-
-
-# Creates a new non-composing 'generator' class instance and registers it.
-# Returns the created instance. Rationale: the instance is returned so that it
-# is possible to first register a generator and then call its 'run' method,
-# bypassing the whole generator selection process.
-#
-rule register-standard ( id : source-types * : target-types + : requirements * )
-{
- local g = [ new generator $(id) : $(source-types) : $(target-types) :
- $(requirements) ] ;
- register $(g) ;
- return $(g) ;
-}
-
-
-# Creates a new composing 'generator' class instance and registers it.
-#
-rule register-composing ( id : source-types * : target-types + : requirements *
- )
-{
- local g = [ new generator $(id) true : $(source-types) : $(target-types) :
- $(requirements) ] ;
- register $(g) ;
- return $(g) ;
-}
-
-
-# Returns all generators belonging to the given 'toolset', i.e. whose ids are
-# '$(toolset).<something>'.
-#
-rule generators-for-toolset ( toolset )
-{
- return $(.generators-for-toolset.$(toolset)) ;
-}
-
-
-# Make generator 'overrider-id' be preferred to 'overridee-id'. If, when
-# searching for generators that could produce a target of a certain type, both
-# those generators are among viable generators, the overridden generator is
-# immediately discarded.
-#
-# The overridden generators are discarded immediately after computing the list
-# of viable generators but before running any of them.
-#
-rule override ( overrider-id : overridee-id )
-{
- .override.$(overrider-id) += $(overridee-id) ;
-}
-
-
-# Returns a list of source type which can possibly be converted to 'target-type'
-# by some chain of generator invocation.
-#
-# More formally, takes all generators for 'target-type' and returns a union of
-# source types for those generators and result of calling itself recursively on
-# source types.
-#
-# Returns '*' in case any type should be considered a viable source type for the
-# given type.
-#
-local rule viable-source-types-real ( target-type )
-{
- local result ;
-
- # 't0' is the initial list of target types we need to process to get a list
- # of their viable source target types. New target types will not be added to
- # this list.
- local t0 = [ type.all-bases $(target-type) ] ;
-
- # 't' is the list of target types which have not yet been processed to get a
- # list of their viable source target types. This list will get expanded as
- # we locate more target types to process.
- local t = $(t0) ;
-
- while $(t)
- {
- # Find all generators for the current type. Unlike
- # 'find-viable-generators' we do not care about the property-set.
- local generators = $(.generators.$(t[1])) ;
- t = $(t[2-]) ;
-
- while $(generators)
- {
- local g = $(generators[1]) ;
- generators = $(generators[2-]) ;
-
- if ! [ $(g).source-types ]
- {
- # Empty source types -- everything can be accepted.
- result = * ;
- # This will terminate this loop.
- generators = ;
- # This will terminate the outer loop.
- t = ;
- }
-
- for local source-type in [ $(g).source-types ]
- {
- if ! $(source-type) in $(result)
- {
- # If a generator accepts a 'source-type' it will also
- # happily accept any type derived from it.
- for local n in [ type.all-derived $(source-type) ]
- {
- if ! $(n) in $(result)
- {
- # Here there is no point in adding target types to
- # the list of types to process in case they are or
- # have already been on that list. We optimize this
- # check by realizing that we only need to avoid the
- # original target type's base types. Other target
- # types that are or have been on the list of target
- # types to process have been added to the 'result'
- # list as well and have thus already been eliminated
- # by the previous if.
- if ! $(n) in $(t0)
- {
- t += $(n) ;
- }
- result += $(n) ;
- }
- }
- }
- }
- }
- }
-
- return $(result) ;
-}
-
-
-# Helper rule, caches the result of 'viable-source-types-real'.
-#
-rule viable-source-types ( target-type )
-{
- local key = .vst.$(target-type) ;
- if ! $($(key))
- {
- .vst-cached-types += $(target-type) ;
- local v = [ viable-source-types-real $(target-type) ] ;
- if ! $(v)
- {
- v = none ;
- }
- $(key) = $(v) ;
- }
-
- if $($(key)) != none
- {
- return $($(key)) ;
- }
-}
-
-
-# Returns the list of source types, which, when passed to 'run' method of
-# 'generator', has some change of being eventually used (probably after
-# conversion by other generators).
-#
-# Returns '*' in case any type should be considered a viable source type for the
-# given generator.
-#
-rule viable-source-types-for-generator-real ( generator )
-{
- local source-types = [ $(generator).source-types ] ;
- if ! $(source-types)
- {
- # If generator does not specify any source types, it might be a special
- # generator like builtin.lib-generator which just relays to other
- # generators. Return '*' to indicate that any source type is possibly
- # OK, since we do not know for sure.
- return * ;
- }
- else
- {
- local result ;
- while $(source-types)
- {
- local s = $(source-types[1]) ;
- source-types = $(source-types[2-]) ;
- local viable-sources = [ generators.viable-source-types $(s) ] ;
- if $(viable-sources) = *
- {
- result = * ;
- source-types = ; # Terminate the loop.
- }
- else
- {
- result += [ type.all-derived $(s) ] $(viable-sources) ;
- }
- }
- return [ sequence.unique $(result) ] ;
- }
-}
-
-
-# Helper rule, caches the result of 'viable-source-types-for-generator'.
-#
-local rule viable-source-types-for-generator ( generator )
-{
- local key = .vstg.$(generator) ;
- if ! $($(key))
- {
- .vstg-cached-generators += $(generator) ;
- local v = [ viable-source-types-for-generator-real $(generator) ] ;
- if ! $(v)
- {
- v = none ;
- }
- $(key) = $(v) ;
- }
-
- if $($(key)) != none
- {
- return $($(key)) ;
- }
-}
-
-
-# Returns usage requirements + list of created targets.
-#
-local rule try-one-generator-really ( project name ? : generator : target-type
- : property-set : sources * )
-{
- local targets =
- [ $(generator).run $(project) $(name) : $(property-set) : $(sources) ] ;
-
- local usage-requirements ;
- local success ;
-
- generators.dout [ indent ] returned $(targets) ;
-
- if $(targets)
- {
- success = true ;
-
- if [ class.is-a $(targets[1]) : property-set ]
- {
- usage-requirements = $(targets[1]) ;
- targets = $(targets[2-]) ;
- }
- else
- {
- usage-requirements = [ property-set.empty ] ;
- }
- }
-
- generators.dout [ indent ] " generator" [ $(generator).id ] " spawned " ;
- generators.dout [ indent ] " " $(targets) ;
- if $(usage-requirements)
- {
- generators.dout [ indent ] " with usage requirements:" $(x) ;
- }
-
- if $(success)
- {
- return $(usage-requirements) $(targets) ;
- }
-}
-
-
-# Checks if generator invocation can be pruned, because it is guaranteed to
-# fail. If so, quickly returns an empty list. Otherwise, calls
-# try-one-generator-really.
-#
-local rule try-one-generator ( project name ? : generator : target-type
- : property-set : sources * )
-{
- local source-types ;
- for local s in $(sources)
- {
- source-types += [ $(s).type ] ;
- }
- local viable-source-types = [ viable-source-types-for-generator $(generator)
- ] ;
-
- if $(source-types) && $(viable-source-types) != * &&
- ! [ set.intersection $(source-types) : $(viable-source-types) ]
- {
- local id = [ $(generator).id ] ;
- generators.dout [ indent ] " ** generator '$(id)' pruned" ;
- #generators.dout [ indent ] "source-types" '$(source-types)' ;
- #generators.dout [ indent ] "viable-source-types" '$(viable-source-types)' ;
- }
- else
- {
- return [ try-one-generator-really $(project) $(name) : $(generator) :
- $(target-type) : $(property-set) : $(sources) ] ;
- }
-}
-
-
-rule construct-types ( project name ? : target-types + : property-set
- : sources + )
-{
- local result ;
- local usage-requirements = [ property-set.empty ] ;
- for local t in $(target-types)
- {
- local r = [ construct $(project) $(name) : $(t) : $(property-set) :
- $(sources) ] ;
- if $(r)
- {
- usage-requirements = [ $(usage-requirements).add $(r[1]) ] ;
- result += $(r[2-]) ;
- }
- }
- # TODO: have to introduce parameter controlling if several types can be
- # matched and add appropriate checks.
-
- # TODO: need to review the documentation for 'construct' to see if it should
- # return $(source) even if nothing can be done with it. Currents docs seem
- # to imply that, contrary to the behaviour.
- if $(result)
- {
- return $(usage-requirements) $(result) ;
- }
- else
- {
- return $(usage-requirements) $(sources) ;
- }
-}
-
-
-# Ensures all 'targets' have their type. If this is not so, exists with error.
-#
-local rule ensure-type ( targets * )
-{
- for local t in $(targets)
- {
- if ! [ $(t).type ]
- {
- import errors ;
- errors.error "target" [ $(t).str ] "has no type" ;
- }
- }
-}
-
-
-# Returns generators which can be used to construct target of specified type
-# with specified properties. Uses the following algorithm:
-# - iterates over requested target-type and all its bases (in the order returned
-# by type.all-bases).
-# - for each type find all generators that generate that type and whose
-# requirements are satisfied by properties.
-# - if the set of generators is not empty, returns that set.
-#
-# Note: this algorithm explicitly ignores generators for base classes if there
-# is at least one generator for the requested target-type.
-#
-local rule find-viable-generators-aux ( target-type : property-set )
-{
- # Select generators that can create the required target type.
- local viable-generators = ;
-
- import type ;
- local t = $(target-type) ;
-
- if $(.debug)
- {
- generators.dout [ indent ] find-viable-generators target-type= $(target-type)
- property-set= [ $(property-set).as-path ] ;
- generators.dout [ indent ] "trying type" $(target-type) ;
- }
-
- local generators = $(.generators.$(target-type)) ;
- if $(generators)
- {
- if $(.debug)
- {
- generators.dout [ indent ] "there are generators for this type" ;
- }
- }
- else
- {
- local t = [ type.base $(target-type) ] ;
-
- # Get the list of generators for the requested type. If no generator is
- # registered, try base type, and so on.
- while $(t)
- {
- if $(.debug)
- {
- generators.dout [ indent ] "trying type" $(t) ;
- }
- if $(.generators.$(t))
- {
- generators.dout [ indent ] "there are generators for this type" ;
- generators = $(.generators.$(t)) ;
-
- # We are here because there were no generators found for
- # target-type but there are some generators for its base type.
- # We will try to use them, but they will produce targets of
- # base type, not of 'target-type'. So, we clone the generators
- # and modify the list of target types.
- local generators2 ;
- for local g in $(generators)
- {
- # generators.register adds a generator to the list of
- # generators for toolsets, which is a bit strange, but
- # should work. That list is only used when inheriting a
- # toolset, which should have been done before running
- # generators.
- generators2 += [ $(g).clone-and-change-target-type $(t) :
- $(target-type) ] ;
- generators.register $(generators2[-1]) ;
- }
- generators = $(generators2) ;
- t = ;
- }
- else
- {
- t = [ type.base $(t) ] ;
- }
- }
- }
-
- for local g in $(generators)
- {
- if $(.debug)
- {
- generators.dout [ indent ] "trying generator" [ $(g).id ] "(" [ $(g).source-types ] -> [ $(g).target-types ] ")" ;
- }
-
- if [ $(g).match-rank $(property-set) ]
- {
- if $(.debug)
- {
- generators.dout [ indent ] " is viable" ;
- }
- viable-generators += $(g) ;
- }
- }
-
- return $(viable-generators) ;
-}
-
-
-rule find-viable-generators ( target-type : property-set )
-{
- local key = $(target-type).$(property-set) ;
- local l = $(.fv.$(key)) ;
- if ! $(l)
- {
- l = [ find-viable-generators-aux $(target-type) : $(property-set) ] ;
- if ! $(l)
- {
- l = none ;
- }
- .fv.$(key) = $(l) ;
- }
-
- if $(l) = none
- {
- l = ;
- }
-
- local viable-generators ;
- for local g in $(l)
- {
- # Avoid trying the same generator twice on different levels.
- if ! $(g) in $(.active-generators)
- {
- viable-generators += $(g) ;
- }
- else
- {
- generators.dout [ indent ] " generator " [ $(g).id ] "is active, discaring" ;
- }
- }
-
- # Generators which override 'all'.
- local all-overrides ;
- # Generators which are overriden.
- local overriden-ids ;
- for local g in $(viable-generators)
- {
- local id = [ $(g).id ] ;
- local this-overrides = $(.override.$(id)) ;
- overriden-ids += $(this-overrides) ;
- if all in $(this-overrides)
- {
- all-overrides += $(g) ;
- }
- }
- if $(all-overrides)
- {
- viable-generators = $(all-overrides) ;
- }
- local result ;
- for local g in $(viable-generators)
- {
- if ! [ $(g).id ] in $(overriden-ids)
- {
- result += $(g) ;
- }
- }
-
- return $(result) ;
-}
-
-
-.construct-stack = ;
-
-
-# Attempts to construct a target by finding viable generators, running them and
-# selecting the dependency graph.
-#
-local rule construct-really ( project name ? : target-type : property-set :
- sources * )
-{
- viable-generators = [ find-viable-generators $(target-type) :
- $(property-set) ] ;
-
- generators.dout [ indent ] "*** " [ sequence.length $(viable-generators) ]
- " viable generators" ;
-
- local result ;
- local generators-that-succeeded ;
- for local g in $(viable-generators)
- {
- # This variable will be restored on exit from this scope.
- local .active-generators = $(g) $(.active-generators) ;
-
- local r = [ try-one-generator $(project) $(name) : $(g) : $(target-type)
- : $(property-set) : $(sources) ] ;
-
- if $(r)
- {
- generators-that-succeeded += $(g) ;
- if $(result)
- {
- ECHO "Error: ambiguity found when searching for best transformation" ;
- ECHO "Trying to produce type '$(target-type)' from: " ;
- for local s in $(sources)
- {
- ECHO " - " [ $(s).str ] ;
- }
- ECHO "Generators that succeeded:" ;
- for local g in $(generators-that-succeeded)
- {
- ECHO " - " [ $(g).id ] ;
- }
- ECHO "First generator produced: " ;
- for local t in $(result[2-])
- {
- ECHO " - " [ $(t).str ] ;
- }
- ECHO "Second generator produced: " ;
- for local t in $(r[2-])
- {
- ECHO " - " [ $(t).str ] ;
- }
- EXIT ;
- }
- else
- {
- result = $(r) ;
- }
- }
- }
-
- return $(result) ;
-}
-
-
-# Attempts to create a target of 'target-type' with 'properties' from 'sources'.
-# The 'sources' are treated as a collection of *possible* ingridients, i.e.
-# there is no obligation to consume them all.
-#
-# Returns a list of targets. When this invocation is first instance of
-# 'construct' in stack, returns only targets of requested 'target-type',
-# otherwise, returns also unused sources and additionally generated targets.
-#
-# If 'top-level' is set, does not suppress generators that are already
-# used in the stack. This may be useful in cases where a generator
-# has to build a metatargets -- for example a target corresponding to
-# built tool.
-#
-rule construct ( project name ? : target-type : property-set * : sources * : top-level ? )
-{
- local saved-stack ;
- if $(top-level)
- {
- saved-active = $(.active-generators) ;
- .active-generators = ;
- }
-
- if (.construct-stack)
- {
- ensure-type $(sources) ;
- }
-
- .construct-stack += 1 ;
-
- increase-indent ;
-
- if $(.debug)
- {
- generators.dout [ indent ] "*** construct" $(target-type) ;
-
- for local s in $(sources)
- {
- generators.dout [ indent ] " from" $(s) ;
- }
- generators.dout [ indent ] " properties:" [ $(property-set).raw ] ;
- }
-
- local result = [ construct-really $(project) $(name) : $(target-type) :
- $(property-set) : $(sources) ] ;
-
- decrease-indent ;
-
- .construct-stack = $(.construct-stack[2-]) ;
-
- if $(top-level)
- {
- .active-generators = $(saved-active) ;
- }
-
- return $(result) ;
-}
-
-# Given 'result', obtained from some generator or generators.construct, adds
-# 'raw-properties' as usage requirements to it. If result already contains usage
-# requirements -- that is the first element of result of an instance of the
-# property-set class, the existing usage requirements and 'raw-properties' are
-# combined.
-#
-rule add-usage-requirements ( result * : raw-properties * )
-{
- if $(result)
- {
- if [ class.is-a $(result[1]) : property-set ]
- {
- return [ $(result[1]).add-raw $(raw-properties) ] $(result[2-]) ;
- }
- else
- {
- return [ property-set.create $(raw-properties) ] $(result) ;
- }
- }
-}
-
-rule dump ( )
-{
- for local g in $(.all-generators)
- {
- ECHO [ $(g).id ] ":" [ $(g).source-types ] -> [ $(g).target-types ] ;
- }
-}
-
[23/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/function.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/function.h b/ext/kenlm/jam-files/engine/function.h
deleted file mode 100644
index 64f26b3..0000000
--- a/ext/kenlm/jam-files/engine/function.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Copyright 2011 Steven Watanabe
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#ifndef FUNCTION_SW20111123_H
-#define FUNCTION_SW20111123_H
-
-#include "object.h"
-#include "frames.h"
-#include "lists.h"
-#include "parse.h"
-#include "strings.h"
-
-typedef struct _function FUNCTION;
-typedef struct _stack STACK;
-
-STACK * stack_global( void );
-void stack_push( STACK * s, LIST * l );
-LIST * stack_pop( STACK * s );
-
-FUNCTION * function_compile( PARSE * parse );
-FUNCTION * function_builtin( LIST * ( * func )( FRAME * frame, int flags ), int flags, const char * * args );
-void function_refer( FUNCTION * );
-void function_free( FUNCTION * );
-OBJECT * function_rulename( FUNCTION * );
-void function_set_rulename( FUNCTION *, OBJECT * );
-void function_location( FUNCTION *, OBJECT * *, int * );
-LIST * function_run( FUNCTION * function, FRAME * frame, STACK * s );
-
-FUNCTION * function_compile_actions( const char * actions, OBJECT * file, int line );
-void function_run_actions( FUNCTION * function, FRAME * frame, STACK * s, string * out );
-
-FUNCTION * function_bind_variables( FUNCTION * f, module_t * module, int * counter );
-FUNCTION * function_unbind_variables( FUNCTION * f );
-
-void function_done( void );
-
-#ifdef HAVE_PYTHON
-
-FUNCTION * function_python( PyObject * function, PyObject * bjam_signature );
-
-#endif
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/glob.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/glob.c b/ext/kenlm/jam-files/engine/glob.c
deleted file mode 100644
index bec00ee..0000000
--- a/ext/kenlm/jam-files/engine/glob.c
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Copyright 1994 Christopher Seiwald. All rights reserved.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/*
- * glob.c - match a string against a simple pattern
- *
- * Understands the following patterns:
- *
- * * any number of characters
- * ? any single character
- * [a-z] any single character in the range a-z
- * [^a-z] any single character not in the range a-z
- * \x match x
- *
- * External functions:
- *
- * glob() - match a string against a simple pattern
- *
- * Internal functions:
- *
- * globchars() - build a bitlist to check for character group match
- */
-
-# include "jam.h"
-
-# define CHECK_BIT( tab, bit ) ( tab[ (bit)/8 ] & (1<<( (bit)%8 )) )
-# define BITLISTSIZE 16 /* bytes used for [chars] in compiled expr */
-
-static void globchars( const char * s, const char * e, char * b );
-
-
-/*
- * glob() - match a string against a simple pattern.
- */
-
-int glob( const char * c, const char * s )
-{
- char bitlist[ BITLISTSIZE ];
- const char * here;
-
- for ( ; ; )
- switch ( *c++ )
- {
- case '\0':
- return *s ? -1 : 0;
-
- case '?':
- if ( !*s++ )
- return 1;
- break;
-
- case '[':
- /* Scan for matching ]. */
-
- here = c;
- do if ( !*c++ ) return 1;
- while ( ( here == c ) || ( *c != ']' ) );
- ++c;
-
- /* Build character class bitlist. */
-
- globchars( here, c, bitlist );
-
- if ( !CHECK_BIT( bitlist, *(const unsigned char *)s ) )
- return 1;
- ++s;
- break;
-
- case '*':
- here = s;
-
- while ( *s )
- ++s;
-
- /* Try to match the rest of the pattern in a recursive */
- /* call. If the match fails we'll back up chars, retrying. */
-
- while ( s != here )
- {
- int r;
-
- /* A fast path for the last token in a pattern. */
- r = *c ? glob( c, s ) : *s ? -1 : 0;
-
- if ( !r )
- return 0;
- if ( r < 0 )
- return 1;
- --s;
- }
- break;
-
- case '\\':
- /* Force literal match of next char. */
- if ( !*c || ( *s++ != *c++ ) )
- return 1;
- break;
-
- default:
- if ( *s++ != c[ -1 ] )
- return 1;
- break;
- }
-}
-
-
-/*
- * globchars() - build a bitlist to check for character group match.
- */
-
-static void globchars( const char * s, const char * e, char * b )
-{
- int neg = 0;
-
- memset( b, '\0', BITLISTSIZE );
-
- if ( *s == '^' )
- {
- ++neg;
- ++s;
- }
-
- while ( s < e )
- {
- int c;
-
- if ( ( s + 2 < e ) && ( s[1] == '-' ) )
- {
- for ( c = s[0]; c <= s[2]; ++c )
- b[ c/8 ] |= ( 1 << ( c % 8 ) );
- s += 3;
- }
- else
- {
- c = *s++;
- b[ c/8 ] |= ( 1 << ( c % 8 ) );
- }
- }
-
- if ( neg )
- {
- int i;
- for ( i = 0; i < BITLISTSIZE; ++i )
- b[ i ] ^= 0377;
- }
-
- /* Do not include \0 in either $[chars] or $[^chars]. */
- b[0] &= 0376;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/hash.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/hash.c b/ext/kenlm/jam-files/engine/hash.c
deleted file mode 100644
index 36f8366..0000000
--- a/ext/kenlm/jam-files/engine/hash.c
+++ /dev/null
@@ -1,387 +0,0 @@
-/*
- * Copyright 1993, 1995 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/*
- * hash.c - simple in-memory hashing routines
- *
- * External routines:
- * hashinit() - initialize a hash table, returning a handle
- * hashitem() - find a record in the table, and optionally enter a new one
- * hashdone() - free a hash table, given its handle
- *
- * Internal routines:
- * hashrehash() - resize and rebuild hp->tab, the hash table
- */
-
-#include "jam.h"
-#include "hash.h"
-
-#include "compile.h"
-
-#include <assert.h>
-
-/* */
-#define HASH_DEBUG_PROFILE 1
-/* */
-
-/* Header attached to all hash table data items. */
-
-typedef struct item ITEM;
-struct item
-{
- ITEM * next;
-};
-
-#define MAX_LISTS 32
-
-struct hash
-{
- /*
- * the hash table, just an array of item pointers
- */
- struct
- {
- int nel;
- ITEM * * base;
- } tab;
-
- int bloat; /* tab.nel / items.nel */
- int inel; /* initial number of elements */
-
- /*
- * the array of records, maintained by these routines - essentially a
- * microallocator
- */
- struct
- {
- int more; /* how many more ITEMs fit in lists[ list ] */
- ITEM * free; /* free list of items */
- char * next; /* where to put more ITEMs in lists[ list ] */
- int size; /* sizeof( ITEM ) + aligned datalen */
- int nel; /* total ITEMs held by all lists[] */
- int list; /* index into lists[] */
-
- struct
- {
- int nel; /* total ITEMs held by this list */
- char * base; /* base of ITEMs array */
- } lists[ MAX_LISTS ];
- } items;
-
- char const * name; /* just for hashstats() */
-};
-
-static void hashrehash( struct hash * );
-static void hashstat( struct hash * );
-
-static unsigned int hash_keyval( OBJECT * key )
-{
- return object_hash( key );
-}
-
-#define hash_bucket(hp, keyval) ((hp)->tab.base + ((keyval) % (hp)->tab.nel))
-
-#define hash_data_key(data) (*(OBJECT * *)(data))
-#define hash_item_data(item) ((HASHDATA *)((char *)item + sizeof(ITEM)))
-#define hash_item_key(item) (hash_data_key(hash_item_data(item)))
-
-
-#define ALIGNED(x) ((x + sizeof(ITEM) - 1) & ~(sizeof(ITEM) - 1))
-
-/*
- * hashinit() - initialize a hash table, returning a handle
- */
-
-struct hash * hashinit( int datalen, char const * name )
-{
- struct hash * hp = (struct hash *)BJAM_MALLOC( sizeof( *hp ) );
-
- hp->bloat = 3;
- hp->tab.nel = 0;
- hp->tab.base = 0;
- hp->items.more = 0;
- hp->items.free = 0;
- hp->items.size = sizeof( ITEM ) + ALIGNED( datalen );
- hp->items.list = -1;
- hp->items.nel = 0;
- hp->inel = 11; /* 47 */
- hp->name = name;
-
- return hp;
-}
-
-
-/*
- * hash_search() - Find the hash item for the given data.
- *
- * Returns a pointer to a hashed item with the given key. If given a 'previous'
- * pointer, makes it point to the item prior to the found item in the same
- * bucket or to 0 if our item is the first item in its bucket.
- */
-
-static ITEM * hash_search( struct hash * hp, unsigned int keyval,
- OBJECT * keydata, ITEM * * previous )
-{
- ITEM * i = *hash_bucket( hp, keyval );
- ITEM * p = 0;
- for ( ; i; i = i->next )
- {
- if ( object_equal( hash_item_key( i ), keydata ) )
- {
- if ( previous )
- *previous = p;
- return i;
- }
- p = i;
- }
- return 0;
-}
-
-
-/*
- * hash_insert() - insert a record in the table or return the existing one
- */
-
-HASHDATA * hash_insert( struct hash * hp, OBJECT * key, int * found )
-{
- ITEM * i;
- unsigned int keyval = hash_keyval( key );
-
- #ifdef HASH_DEBUG_PROFILE
- profile_frame prof[ 1 ];
- if ( DEBUG_PROFILE )
- profile_enter( 0, prof );
- #endif
-
- if ( !hp->items.more )
- hashrehash( hp );
-
- i = hash_search( hp, keyval, key, 0 );
- if ( i )
- *found = 1;
- else
- {
- ITEM * * base = hash_bucket( hp, keyval );
-
- /* Try to grab one from the free list. */
- if ( hp->items.free )
- {
- i = hp->items.free;
- hp->items.free = i->next;
- assert( !hash_item_key( i ) );
- }
- else
- {
- i = (ITEM *)hp->items.next;
- hp->items.next += hp->items.size;
- }
- --hp->items.more;
- i->next = *base;
- *base = i;
- *found = 0;
- }
-
- #ifdef HASH_DEBUG_PROFILE
- if ( DEBUG_PROFILE )
- profile_exit( prof );
- #endif
-
- return hash_item_data( i );
-}
-
-
-/*
- * hash_find() - find a record in the table or NULL if none exists
- */
-
-HASHDATA * hash_find( struct hash * hp, OBJECT * key )
-{
- ITEM * i;
- unsigned int keyval = hash_keyval( key );
-
- #ifdef HASH_DEBUG_PROFILE
- profile_frame prof[ 1 ];
- if ( DEBUG_PROFILE )
- profile_enter( 0, prof );
- #endif
-
- if ( !hp->items.nel )
- {
- #ifdef HASH_DEBUG_PROFILE
- if ( DEBUG_PROFILE )
- profile_exit( prof );
- #endif
- return 0;
- }
-
- i = hash_search( hp, keyval, key, 0 );
-
- #ifdef HASH_DEBUG_PROFILE
- if ( DEBUG_PROFILE )
- profile_exit( prof );
- #endif
-
- return i ? hash_item_data( i ) : 0;
-}
-
-
-/*
- * hashrehash() - resize and rebuild hp->tab, the hash table
- */
-
-static void hashrehash( struct hash * hp )
-{
- int i = ++hp->items.list;
- hp->items.more = i ? 2 * hp->items.nel : hp->inel;
- hp->items.next = (char *)BJAM_MALLOC( hp->items.more * hp->items.size );
- hp->items.free = 0;
-
- hp->items.lists[ i ].nel = hp->items.more;
- hp->items.lists[ i ].base = hp->items.next;
- hp->items.nel += hp->items.more;
-
- if ( hp->tab.base )
- BJAM_FREE( (char *)hp->tab.base );
-
- hp->tab.nel = hp->items.nel * hp->bloat;
- hp->tab.base = (ITEM * *)BJAM_MALLOC( hp->tab.nel * sizeof( ITEM * * ) );
-
- memset( (char *)hp->tab.base, '\0', hp->tab.nel * sizeof( ITEM * ) );
-
- for ( i = 0; i < hp->items.list; ++i )
- {
- int nel = hp->items.lists[ i ].nel;
- char * next = hp->items.lists[ i ].base;
-
- for ( ; nel--; next += hp->items.size )
- {
- ITEM * i = (ITEM *)next;
- ITEM * * ip = hp->tab.base + object_hash( hash_item_key( i ) ) %
- hp->tab.nel;
- /* code currently assumes rehashing only when there are no free
- * items
- */
- assert( hash_item_key( i ) );
-
- i->next = *ip;
- *ip = i;
- }
- }
-}
-
-
-void hashenumerate( struct hash * hp, void (* f)( void *, void * ), void * data
- )
-{
- int i;
- for ( i = 0; i <= hp->items.list; ++i )
- {
- char * next = hp->items.lists[ i ].base;
- int nel = hp->items.lists[ i ].nel;
- if ( i == hp->items.list )
- nel -= hp->items.more;
-
- for ( ; nel--; next += hp->items.size )
- {
- ITEM * const i = (ITEM *)next;
- if ( hash_item_key( i ) != 0 ) /* Do not enumerate freed items. */
- f( hash_item_data( i ), data );
- }
- }
-}
-
-
-/*
- * hash_free() - free a hash table, given its handle
- */
-
-void hash_free( struct hash * hp )
-{
- int i;
- if ( !hp )
- return;
- if ( hp->tab.base )
- BJAM_FREE( (char *)hp->tab.base );
- for ( i = 0; i <= hp->items.list; ++i )
- BJAM_FREE( hp->items.lists[ i ].base );
- BJAM_FREE( (char *)hp );
-}
-
-
-static void hashstat( struct hash * hp )
-{
- struct hashstats stats[ 1 ];
- hashstats_init( stats );
- hashstats_add( stats, hp );
- hashstats_print( stats, hp->name );
-}
-
-
-void hashstats_init( struct hashstats * stats )
-{
- stats->count = 0;
- stats->num_items = 0;
- stats->tab_size = 0;
- stats->item_size = 0;
- stats->sets = 0;
- stats->num_hashes = 0;
-}
-
-
-void hashstats_add( struct hashstats * stats, struct hash * hp )
-{
- if ( hp )
- {
- ITEM * * tab = hp->tab.base;
- int nel = hp->tab.nel;
- int count = 0;
- int sets = 0;
- int i;
-
- for ( i = 0; i < nel; ++i )
- {
- ITEM * item;
- int here = 0;
- for ( item = tab[ i ]; item; item = item->next )
- ++here;
-
- count += here;
- if ( here > 0 )
- ++sets;
- }
-
- stats->count += count;
- stats->sets += sets;
- stats->num_items += hp->items.nel;
- stats->tab_size += hp->tab.nel;
- stats->item_size = hp->items.size;
- ++stats->num_hashes;
- }
-}
-
-
-void hashstats_print( struct hashstats * stats, char const * name )
-{
- printf( "%s table: %d+%d+%d (%dK+%luK+%luK) items+table+hash, %f density\n",
- name,
- stats->count,
- stats->num_items,
- stats->tab_size,
- stats->num_items * stats->item_size / 1024,
- (long unsigned)stats->tab_size * sizeof( ITEM * * ) / 1024,
- (long unsigned)stats->num_hashes * sizeof( struct hash ) / 1024,
- (float)stats->count / (float)stats->sets );
-}
-
-
-void hashdone( struct hash * hp )
-{
- if ( !hp )
- return;
- if ( DEBUG_MEM || DEBUG_PROFILE )
- hashstat( hp );
- hash_free( hp );
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/hash.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/hash.h b/ext/kenlm/jam-files/engine/hash.h
deleted file mode 100644
index 7c40e8c..0000000
--- a/ext/kenlm/jam-files/engine/hash.h
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Copyright 1993, 1995 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/*
- * hash.h - simple in-memory hashing routines
- */
-
-#ifndef BOOST_JAM_HASH_H
-#define BOOST_JAM_HASH_H
-
-#include "object.h"
-
-/*
- * An opaque struct representing an item in the hash table. The first element of
- * every struct stored in the table must be an OBJECT * which is treated as the
- * key.
- */
-typedef struct hashdata HASHDATA;
-
-/*
- * hashinit() - initialize a hash table, returning a handle.
- *
- * Parameters:
- * datalen - item size
- * name - used for debugging
- */
-struct hash * hashinit( int datalen, char const * name );
-
-/*
- * hash_free() - free a hash table, given its handle
- */
-void hash_free( struct hash * );
-void hashdone( struct hash * );
-
-/*
- * hashenumerate() - call f(i, data) on each item, i in the hash table. The
- * enumeration order is unspecified.
- */
-void hashenumerate( struct hash *, void (* f)( void *, void * ), void * data );
-
-/*
- * hash_insert() - insert a new item in a hash table, or return an existing one.
- *
- * Preconditions:
- * - hp must be a hash table created by hashinit()
- * - key must be an object created by object_new()
- *
- * Postconditions:
- * - if the key does not already exist in the hash table, *found == 0 and the
- * result will be a pointer to an uninitialized item. The key of the new
- * item must be set to a value equal to key before any further operations on
- * the hash table except hashdone().
- * - if the key is present then *found == 1 and the result is a pointer to the
- * existing record.
- */
-HASHDATA * hash_insert( struct hash *, OBJECT * key, int * found );
-
-/*
- * hash_find() - find a record in the table or NULL if none exists
- */
-HASHDATA * hash_find( struct hash *, OBJECT * key );
-
-struct hashstats {
- int count;
- int num_items;
- int tab_size;
- int item_size;
- int sets;
- int num_hashes;
-};
-
-void hashstats_init( struct hashstats * stats );
-void hashstats_add( struct hashstats * stats, struct hash * );
-void hashstats_print( struct hashstats * stats, char const * name );
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/hcache.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/hcache.c b/ext/kenlm/jam-files/engine/hcache.c
deleted file mode 100644
index 3cf15f7..0000000
--- a/ext/kenlm/jam-files/engine/hcache.c
+++ /dev/null
@@ -1,519 +0,0 @@
-/*
- * This file has been donated to Jam.
- */
-
-/*
- * Craig W. McPheeters, Alias|Wavefront.
- *
- * hcache.c hcache.h - handle cacheing of #includes in source files.
- *
- * Create a cache of files scanned for headers. When starting jam, look for the
- * cache file and load it if present. When finished the binding phase, create a
- * new header cache. The cache contains files, their timestamps and the header
- * files found in their scan. During the binding phase of jam, look in the
- * header cache first for the headers contained in a file. If the cache is
- * present and valid, use its contents. This results in dramatic speedups with
- * large projects (e.g. 3min -> 1min startup for one project.)
- *
- * External routines:
- * hcache_init() - read and parse the local .jamdeps file.
- * hcache_done() - write a new .jamdeps file.
- * hcache() - return list of headers on target. Use cache or do a scan.
- *
- * The dependency file format is an ASCII file with 1 line per target. Each line
- * has the following fields:
- * @boundname@ timestamp_sec timestamp_nsec @file@ @file@ @file@ ...
- */
-
-#ifdef OPT_HEADER_CACHE_EXT
-
-#include "jam.h"
-#include "hcache.h"
-
-#include "hash.h"
-#include "headers.h"
-#include "lists.h"
-#include "modules.h"
-#include "object.h"
-#include "parse.h"
-#include "regexp.h"
-#include "rules.h"
-#include "search.h"
-#include "timestamp.h"
-#include "variable.h"
-
-typedef struct hcachedata HCACHEDATA ;
-
-struct hcachedata
-{
- OBJECT * boundname;
- timestamp time;
- LIST * includes;
- LIST * hdrscan; /* the HDRSCAN value for this target */
- int age; /* if too old, we will remove it from cache */
- HCACHEDATA * next;
-};
-
-
-static struct hash * hcachehash = 0;
-static HCACHEDATA * hcachelist = 0;
-
-static int queries = 0;
-static int hits = 0;
-
-#define CACHE_FILE_VERSION "version 5"
-#define CACHE_RECORD_HEADER "header"
-#define CACHE_RECORD_END "end"
-
-
-/*
- * Return the name of the header cache file. May return NULL.
- *
- * The user sets this by setting the HCACHEFILE variable in a Jamfile. We cache
- * the result so the user can not change the cache file during header scanning.
- */
-
-static const char * cache_name( void )
-{
- static OBJECT * name = 0;
- if ( !name )
- {
- LIST * const hcachevar = var_get( root_module(), constant_HCACHEFILE );
-
- if ( !list_empty( hcachevar ) )
- {
- TARGET * const t = bindtarget( list_front( hcachevar ) );
-
- pushsettings( root_module(), t->settings );
- /* Do not expect the cache file to be generated, so pass 0 as the
- * third argument to search. Expect the location to be specified via
- * LOCATE, so pass 0 as the fourth arugment.
- */
- object_free( t->boundname );
- t->boundname = search( t->name, &t->time, 0, 0 );
- popsettings( root_module(), t->settings );
-
- name = object_copy( t->boundname );
- }
- }
- return name ? object_str( name ) : 0;
-}
-
-
-/*
- * Return the maximum age a cache entry can have before it is purged from the
- * cache.
- */
-
-static int cache_maxage( void )
-{
- int age = 100;
- LIST * const var = var_get( root_module(), constant_HCACHEMAXAGE );
- if ( !list_empty( var ) )
- {
- age = atoi( object_str( list_front( var ) ) );
- if ( age < 0 )
- age = 0;
- }
- return age;
-}
-
-
-/*
- * Read a netstring. The caveat is that the string can not contain ASCII 0. The
- * returned value is as returned by object_new().
- */
-
-OBJECT * read_netstring( FILE * f )
-{
- unsigned long len;
- static char * buf = NULL;
- static unsigned long buf_len = 0;
-
- if ( fscanf( f, " %9lu", &len ) != 1 )
- return NULL;
- if ( fgetc( f ) != (int)'\t' )
- return NULL;
-
- if ( len > 1024 * 64 )
- return NULL; /* sanity check */
-
- if ( len > buf_len )
- {
- unsigned long new_len = buf_len * 2;
- if ( new_len < len )
- new_len = len;
- buf = (char *)BJAM_REALLOC( buf, new_len + 1 );
- if ( buf )
- buf_len = new_len;
- }
-
- if ( !buf )
- return NULL;
-
- if ( fread( buf, 1, len, f ) != len )
- return NULL;
- if ( fgetc( f ) != (int)'\n' )
- return NULL;
-
- buf[ len ] = 0;
- return object_new( buf );
-}
-
-
-/*
- * Write a netstring.
- */
-
-void write_netstring( FILE * f, char const * s )
-{
- if ( !s )
- s = "";
- fprintf( f, "%lu\t%s\n", (long unsigned)strlen( s ), s );
-}
-
-
-void hcache_init()
-{
- FILE * f;
- OBJECT * version = 0;
- int header_count = 0;
- const char * hcachename;
-
- if ( hcachehash )
- return;
-
- hcachehash = hashinit( sizeof( HCACHEDATA ), "hcache" );
-
- if ( !( hcachename = cache_name() ) )
- return;
-
- if ( !( f = fopen( hcachename, "rb" ) ) )
- return;
-
- version = read_netstring( f );
-
- if ( !version || strcmp( object_str( version ), CACHE_FILE_VERSION ) )
- goto bail;
-
- while ( 1 )
- {
- HCACHEDATA cachedata;
- HCACHEDATA * c;
- OBJECT * record_type = 0;
- OBJECT * time_secs_str = 0;
- OBJECT * time_nsecs_str = 0;
- OBJECT * age_str = 0;
- OBJECT * includes_count_str = 0;
- OBJECT * hdrscan_count_str = 0;
- int i;
- int count;
- LIST * l;
- int found;
-
- cachedata.boundname = 0;
- cachedata.includes = 0;
- cachedata.hdrscan = 0;
-
- record_type = read_netstring( f );
- if ( !record_type )
- {
- fprintf( stderr, "invalid %s\n", hcachename );
- goto cleanup;
- }
- if ( !strcmp( object_str( record_type ), CACHE_RECORD_END ) )
- {
- object_free( record_type );
- break;
- }
- if ( strcmp( object_str( record_type ), CACHE_RECORD_HEADER ) )
- {
- fprintf( stderr, "invalid %s with record separator <%s>\n",
- hcachename, record_type ? object_str( record_type ) : "<null>" );
- goto cleanup;
- }
-
- cachedata.boundname = read_netstring( f );
- time_secs_str = read_netstring( f );
- time_nsecs_str = read_netstring( f );
- age_str = read_netstring( f );
- includes_count_str = read_netstring( f );
-
- if ( !cachedata.boundname || !time_secs_str || !time_nsecs_str ||
- !age_str || !includes_count_str )
- {
- fprintf( stderr, "invalid %s\n", hcachename );
- goto cleanup;
- }
-
- timestamp_init( &cachedata.time, atoi( object_str( time_secs_str ) ),
- atoi( object_str( time_nsecs_str ) ) );
- cachedata.age = atoi( object_str( age_str ) ) + 1;
-
- count = atoi( object_str( includes_count_str ) );
- for ( l = L0, i = 0; i < count; ++i )
- {
- OBJECT * const s = read_netstring( f );
- if ( !s )
- {
- fprintf( stderr, "invalid %s\n", hcachename );
- list_free( l );
- goto cleanup;
- }
- l = list_push_back( l, s );
- }
- cachedata.includes = l;
-
- hdrscan_count_str = read_netstring( f );
- if ( !hdrscan_count_str )
- {
- fprintf( stderr, "invalid %s\n", hcachename );
- goto cleanup;
- }
-
- count = atoi( object_str( hdrscan_count_str ) );
- for ( l = L0, i = 0; i < count; ++i )
- {
- OBJECT * const s = read_netstring( f );
- if ( !s )
- {
- fprintf( stderr, "invalid %s\n", hcachename );
- list_free( l );
- goto cleanup;
- }
- l = list_push_back( l, s );
- }
- cachedata.hdrscan = l;
-
- c = (HCACHEDATA *)hash_insert( hcachehash, cachedata.boundname, &found )
- ;
- if ( !found )
- {
- c->boundname = cachedata.boundname;
- c->includes = cachedata.includes;
- c->hdrscan = cachedata.hdrscan;
- c->age = cachedata.age;
- timestamp_copy( &c->time, &cachedata.time );
- }
- else
- {
- fprintf( stderr, "can not insert header cache item, bailing on %s"
- "\n", hcachename );
- goto cleanup;
- }
-
- c->next = hcachelist;
- hcachelist = c;
-
- ++header_count;
-
- object_free( record_type );
- object_free( time_secs_str );
- object_free( time_nsecs_str );
- object_free( age_str );
- object_free( includes_count_str );
- object_free( hdrscan_count_str );
- continue;
-
-cleanup:
-
- if ( record_type ) object_free( record_type );
- if ( time_secs_str ) object_free( time_secs_str );
- if ( time_nsecs_str ) object_free( time_nsecs_str );
- if ( age_str ) object_free( age_str );
- if ( includes_count_str ) object_free( includes_count_str );
- if ( hdrscan_count_str ) object_free( hdrscan_count_str );
-
- if ( cachedata.boundname ) object_free( cachedata.boundname );
- if ( cachedata.includes ) list_free( cachedata.includes );
- if ( cachedata.hdrscan ) list_free( cachedata.hdrscan );
-
- goto bail;
- }
-
- if ( DEBUG_HEADER )
- printf( "hcache read from file %s\n", hcachename );
-
-bail:
- if ( version )
- object_free( version );
- fclose( f );
-}
-
-
-void hcache_done()
-{
- FILE * f;
- HCACHEDATA * c;
- int header_count = 0;
- const char * hcachename;
- int maxage;
-
- if ( !hcachehash )
- return;
-
- if ( !( hcachename = cache_name() ) )
- goto cleanup;
-
- if ( !( f = fopen( hcachename, "wb" ) ) )
- goto cleanup;
-
- maxage = cache_maxage();
-
- /* Print out the version. */
- write_netstring( f, CACHE_FILE_VERSION );
-
- c = hcachelist;
- for ( c = hcachelist; c; c = c->next )
- {
- LISTITER iter;
- LISTITER end;
- char time_secs_str[ 30 ];
- char time_nsecs_str[ 30 ];
- char age_str[ 30 ];
- char includes_count_str[ 30 ];
- char hdrscan_count_str[ 30 ];
-
- if ( maxage == 0 )
- c->age = 0;
- else if ( c->age > maxage )
- continue;
-
- sprintf( includes_count_str, "%lu", (long unsigned)list_length(
- c->includes ) );
- sprintf( hdrscan_count_str, "%lu", (long unsigned)list_length(
- c->hdrscan ) );
- sprintf( time_secs_str, "%lu", (long unsigned)c->time.secs );
- sprintf( time_nsecs_str, "%lu", (long unsigned)c->time.nsecs );
- sprintf( age_str, "%lu", (long unsigned)c->age );
-
- write_netstring( f, CACHE_RECORD_HEADER );
- write_netstring( f, object_str( c->boundname ) );
- write_netstring( f, time_secs_str );
- write_netstring( f, time_nsecs_str );
- write_netstring( f, age_str );
- write_netstring( f, includes_count_str );
- for ( iter = list_begin( c->includes ), end = list_end( c->includes );
- iter != end; iter = list_next( iter ) )
- write_netstring( f, object_str( list_item( iter ) ) );
- write_netstring( f, hdrscan_count_str );
- for ( iter = list_begin( c->hdrscan ), end = list_end( c->hdrscan );
- iter != end; iter = list_next( iter ) )
- write_netstring( f, object_str( list_item( iter ) ) );
- fputs( "\n", f );
- ++header_count;
- }
- write_netstring( f, CACHE_RECORD_END );
-
- if ( DEBUG_HEADER )
- printf( "hcache written to %s. %d dependencies, %.0f%% hit rate\n",
- hcachename, header_count, queries ? 100.0 * hits / queries : 0 );
-
- fclose ( f );
-
-cleanup:
- for ( c = hcachelist; c; c = c->next )
- {
- list_free( c->includes );
- list_free( c->hdrscan );
- object_free( c->boundname );
- }
-
- hcachelist = 0;
- if ( hcachehash )
- hashdone( hcachehash );
- hcachehash = 0;
-}
-
-
-LIST * hcache( TARGET * t, int rec, regexp * re[], LIST * hdrscan )
-{
- HCACHEDATA * c;
-
- ++queries;
-
- if ( ( c = (HCACHEDATA *)hash_find( hcachehash, t->boundname ) ) )
- {
- if ( !timestamp_cmp( &c->time, &t->time ) )
- {
- LIST * const l1 = hdrscan;
- LIST * const l2 = c->hdrscan;
- LISTITER iter1 = list_begin( l1 );
- LISTITER const end1 = list_end( l1 );
- LISTITER iter2 = list_begin( l2 );
- LISTITER const end2 = list_end( l2 );
- while ( iter1 != end1 && iter2 != end2 )
- {
- if ( !object_equal( list_item( iter1 ), list_item( iter2 ) ) )
- iter1 = end1;
- else
- {
- iter1 = list_next( iter1 );
- iter2 = list_next( iter2 );
- }
- }
- if ( iter1 != end1 || iter2 != end2 )
- {
- if ( DEBUG_HEADER )
- {
- printf( "HDRSCAN out of date in cache for %s\n",
- object_str( t->boundname ) );
- printf(" real : ");
- list_print( hdrscan );
- printf( "\n cached: " );
- list_print( c->hdrscan );
- printf( "\n" );
- }
-
- list_free( c->includes );
- list_free( c->hdrscan );
- c->includes = L0;
- c->hdrscan = L0;
- }
- else
- {
- if ( DEBUG_HEADER )
- printf( "using header cache for %s\n", object_str(
- t->boundname ) );
- c->age = 0;
- ++hits;
- return list_copy( c->includes );
- }
- }
- else
- {
- if ( DEBUG_HEADER )
- printf ("header cache out of date for %s\n", object_str(
- t->boundname ) );
- list_free( c->includes );
- list_free( c->hdrscan );
- c->includes = L0;
- c->hdrscan = L0;
- }
- }
- else
- {
- int found;
- c = (HCACHEDATA *)hash_insert( hcachehash, t->boundname, &found );
- if ( !found )
- {
- c->boundname = object_copy( t->boundname );
- c->next = hcachelist;
- hcachelist = c;
- }
- }
-
- /* 'c' points at the cache entry. Its out of date. */
- {
- LIST * const l = headers1( L0, t->boundname, rec, re );
-
- timestamp_copy( &c->time, &t->time );
- c->age = 0;
- c->includes = list_copy( l );
- c->hdrscan = list_copy( hdrscan );
-
- return l;
- }
-}
-
-#endif /* OPT_HEADER_CACHE_EXT */
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/hcache.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/hcache.h b/ext/kenlm/jam-files/engine/hcache.h
deleted file mode 100644
index a9d929d..0000000
--- a/ext/kenlm/jam-files/engine/hcache.h
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * This file is not part of Jam
- */
-
-/*
- * hcache.h - handle #includes in source files
- */
-#ifndef HCACHE_H
-#define HCACHE_H
-
-#include "lists.h"
-#include "regexp.h"
-#include "rules.h"
-
-void hcache_init( void );
-void hcache_done( void );
-LIST * hcache( TARGET * t, int rec, regexp * re[], LIST * hdrscan );
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/hdrmacro.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/hdrmacro.c b/ext/kenlm/jam-files/engine/hdrmacro.c
deleted file mode 100644
index eb4fe90..0000000
--- a/ext/kenlm/jam-files/engine/hdrmacro.c
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Copyright 1993, 2000 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * hdrmacro.c - handle header files that define macros used in #include
- * statements.
- *
- * we look for lines like "#define MACRO <....>" or '#define MACRO " "' in
- * the target file. When found, we then phony up a rule invocation like:
- *
- * $(HDRRULE) <target> : <resolved included files> ;
- *
- * External routines:
- * headers1() - scan a target for "#include MACRO" lines and try to resolve
- * them when needed
- *
- * Internal routines:
- * headers1() - using regexp, scan a file and build include LIST
- */
-
-#include "jam.h"
-#include "hdrmacro.h"
-
-#include "compile.h"
-#include "hash.h"
-#include "lists.h"
-#include "object.h"
-#include "parse.h"
-#include "rules.h"
-#include "strings.h"
-#include "subst.h"
-#include "variable.h"
-
-
-/* this type is used to store a dictionary of file header macros */
-typedef struct header_macro
-{
- OBJECT * symbol;
- OBJECT * filename; /* we could maybe use a LIST here ?? */
-} HEADER_MACRO;
-
-static struct hash * header_macros_hash = 0;
-
-
-/*
- * headers() - scan a target for include files and call HDRRULE
- */
-
-#define MAXINC 10
-
-void macro_headers( TARGET * t )
-{
- static regexp * re = 0;
- FILE * f;
- char buf[ 1024 ];
-
- if ( DEBUG_HEADER )
- printf( "macro header scan for %s\n", object_str( t->name ) );
-
- /* This regexp is used to detect lines of the form
- * "#define MACRO <....>" or "#define MACRO "....."
- * in the header macro files.
- */
- if ( !re )
- {
- OBJECT * const re_str = object_new(
- "^[ ]*#[ ]*define[ ]*([A-Za-z][A-Za-z0-9_]*)[ ]*"
- "[<\"]([^\">]*)[\">].*$" );
- re = regex_compile( re_str );
- object_free( re_str );
- }
-
- if ( !( f = fopen( object_str( t->boundname ), "r" ) ) )
- return;
-
- while ( fgets( buf, sizeof( buf ), f ) )
- {
- HEADER_MACRO var;
- HEADER_MACRO * v = &var;
-
- if ( regexec( re, buf ) && re->startp[ 1 ] )
- {
- OBJECT * symbol;
- int found;
- /* we detected a line that looks like "#define MACRO filename */
- ( (char *)re->endp[ 1 ] )[ 0 ] = '\0';
- ( (char *)re->endp[ 2 ] )[ 0 ] = '\0';
-
- if ( DEBUG_HEADER )
- printf( "macro '%s' used to define filename '%s' in '%s'\n",
- re->startp[ 1 ], re->startp[ 2 ], object_str( t->boundname )
- );
-
- /* add macro definition to hash table */
- if ( !header_macros_hash )
- header_macros_hash = hashinit( sizeof( HEADER_MACRO ),
- "hdrmacros" );
-
- symbol = object_new( re->startp[ 1 ] );
- v = (HEADER_MACRO *)hash_insert( header_macros_hash, symbol, &found
- );
- if ( !found )
- {
- v->symbol = symbol;
- v->filename = object_new( re->startp[ 2 ] ); /* never freed */
- }
- else
- object_free( symbol );
- /* XXXX: FOR NOW, WE IGNORE MULTIPLE MACRO DEFINITIONS !! */
- /* WE MIGHT AS WELL USE A LIST TO STORE THEM.. */
- }
- }
-
- fclose( f );
-}
-
-
-OBJECT * macro_header_get( OBJECT * macro_name )
-{
- HEADER_MACRO * v;
- if ( header_macros_hash && ( v = (HEADER_MACRO *)hash_find(
- header_macros_hash, macro_name ) ) )
- {
- if ( DEBUG_HEADER )
- printf( "### macro '%s' evaluated to '%s'\n", object_str( macro_name
- ), object_str( v->filename ) );
- return v->filename;
- }
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/hdrmacro.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/hdrmacro.h b/ext/kenlm/jam-files/engine/hdrmacro.h
deleted file mode 100644
index 7595ede..0000000
--- a/ext/kenlm/jam-files/engine/hdrmacro.h
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Copyright 1993, 1995 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/*
- * hdrmacro.h - parses header files for #define MACRO <filename> or
- * #define MACRO "filename" definitions
- */
-
-#ifndef HDRMACRO_SW20111118_H
-#define HDRMACRO_SW20111118_H
-
-#include "object.h"
-#include "rules.h"
-
-void macro_headers( TARGET * );
-OBJECT * macro_header_get( OBJECT * macro_name );
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/headers.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/headers.c b/ext/kenlm/jam-files/engine/headers.c
deleted file mode 100644
index 0d9558d..0000000
--- a/ext/kenlm/jam-files/engine/headers.c
+++ /dev/null
@@ -1,197 +0,0 @@
-/*
- * Copyright 1993, 2000 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * headers.c - handle #includes in source files
- *
- * Using regular expressions provided as the variable $(HDRSCAN), headers()
- * searches a file for #include files and phonies up a rule invocation:
- * $(HDRRULE) <target> : <include files> ;
- *
- * External routines:
- * headers() - scan a target for include files and call HDRRULE
- *
- * Internal routines:
- * headers1() - using regexp, scan a file and build include LIST
- */
-
-#include "jam.h"
-#include "headers.h"
-
-#include "compile.h"
-#include "hdrmacro.h"
-#include "lists.h"
-#include "modules.h"
-#include "object.h"
-#include "parse.h"
-#include "rules.h"
-#include "subst.h"
-#include "variable.h"
-
-#ifdef OPT_HEADER_CACHE_EXT
-# include "hcache.h"
-#endif
-
-#ifndef OPT_HEADER_CACHE_EXT
-static LIST * headers1( LIST *, OBJECT * file, int rec, regexp * re[] );
-#endif
-
-
-/*
- * headers() - scan a target for include files and call HDRRULE
- */
-
-#define MAXINC 10
-
-void headers( TARGET * t )
-{
- LIST * hdrscan;
- LIST * hdrrule;
- #ifndef OPT_HEADER_CACHE_EXT
- LIST * headlist = L0;
- #endif
- regexp * re[ MAXINC ];
- int rec = 0;
- LISTITER iter;
- LISTITER end;
-
- hdrscan = var_get( root_module(), constant_HDRSCAN );
- if ( list_empty( hdrscan ) )
- return;
-
- hdrrule = var_get( root_module(), constant_HDRRULE );
- if ( list_empty( hdrrule ) )
- return;
-
- if ( DEBUG_HEADER )
- printf( "header scan %s\n", object_str( t->name ) );
-
- /* Compile all regular expressions in HDRSCAN */
- iter = list_begin( hdrscan );
- end = list_end( hdrscan );
- for ( ; ( rec < MAXINC ) && iter != end; iter = list_next( iter ) )
- {
- re[ rec++ ] = regex_compile( list_item( iter ) );
- }
-
- /* Doctor up call to HDRRULE rule */
- /* Call headers1() to get LIST of included files. */
- {
- FRAME frame[ 1 ];
- frame_init( frame );
- lol_add( frame->args, list_new( object_copy( t->name ) ) );
-#ifdef OPT_HEADER_CACHE_EXT
- lol_add( frame->args, hcache( t, rec, re, hdrscan ) );
-#else
- lol_add( frame->args, headers1( headlist, t->boundname, rec, re ) );
-#endif
-
- if ( lol_get( frame->args, 1 ) )
- {
- OBJECT * rulename = list_front( hdrrule );
- /* The third argument to HDRRULE is the bound name of $(<). */
- lol_add( frame->args, list_new( object_copy( t->boundname ) ) );
- list_free( evaluate_rule( bindrule( rulename, frame->module ), rulename, frame ) );
- }
-
- /* Clean up. */
- frame_free( frame );
- }
-}
-
-
-/*
- * headers1() - using regexp, scan a file and build include LIST.
- */
-
-#ifndef OPT_HEADER_CACHE_EXT
-static
-#endif
-LIST * headers1( LIST * l, OBJECT * file, int rec, regexp * re[] )
-{
- FILE * f;
- char buf[ 1024 ];
- int i;
- static regexp * re_macros = 0;
-
-#ifdef OPT_IMPROVED_PATIENCE_EXT
- static int count = 0;
- ++count;
- if ( ( ( count == 100 ) || !( count % 1000 ) ) && DEBUG_MAKE )
- {
- printf( "...patience...\n" );
- fflush( stdout );
- }
-#endif
-
- /* The following regexp is used to detect cases where a file is included
- * through a line like "#include MACRO".
- */
- if ( re_macros == 0 )
- {
- OBJECT * const re_str = object_new(
- "#[ \t]*include[ \t]*([A-Za-z][A-Za-z0-9_]*).*$" );
- re_macros = regex_compile( re_str );
- object_free( re_str );
- }
-
- if ( !( f = fopen( object_str( file ), "r" ) ) )
- return l;
-
- while ( fgets( buf, sizeof( buf ), f ) )
- {
- for ( i = 0; i < rec; ++i )
- if ( regexec( re[ i ], buf ) && re[ i ]->startp[ 1 ] )
- {
- ( (char *)re[ i ]->endp[ 1 ] )[ 0 ] = '\0';
- if ( DEBUG_HEADER )
- printf( "header found: %s\n", re[ i ]->startp[ 1 ] );
- l = list_push_back( l, object_new( re[ i ]->startp[ 1 ] ) );
- }
-
- /* Special treatment for #include MACRO. */
- if ( regexec( re_macros, buf ) && re_macros->startp[ 1 ] )
- {
- OBJECT * header_filename;
- OBJECT * macro_name;
-
- ( (char *)re_macros->endp[ 1 ] )[ 0 ] = '\0';
-
- if ( DEBUG_HEADER )
- printf( "macro header found: %s", re_macros->startp[ 1 ] );
-
- macro_name = object_new( re_macros->startp[ 1 ] );
- header_filename = macro_header_get( macro_name );
- object_free( macro_name );
- if ( header_filename )
- {
- if ( DEBUG_HEADER )
- printf( " resolved to '%s'\n", object_str( header_filename )
- );
- l = list_push_back( l, object_copy( header_filename ) );
- }
- else
- {
- if ( DEBUG_HEADER )
- printf( " ignored !!\n" );
- }
- }
- }
-
- fclose( f );
- return l;
-}
-
-
-void regerror( char const * s )
-{
- printf( "re error %s\n", s );
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/headers.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/headers.h b/ext/kenlm/jam-files/engine/headers.h
deleted file mode 100644
index 1c0a642..0000000
--- a/ext/kenlm/jam-files/engine/headers.h
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright 1993, 1995 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/*
- * headers.h - handle #includes in source files
- */
-
-#ifndef HEADERS_SW20111118_H
-#define HEADERS_SW20111118_H
-
-#include "object.h"
-#include "rules.h"
-#include "regexp.h"
-
-void headers( TARGET * t );
-
-#ifdef OPT_HEADER_CACHE_EXT
-struct regexp;
-LIST * headers1( LIST *l, OBJECT * file, int rec, struct regexp *re[] );
-#endif
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/jam.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/jam.c b/ext/kenlm/jam-files/engine/jam.c
deleted file mode 100644
index 1c80eec..0000000
--- a/ext/kenlm/jam-files/engine/jam.c
+++ /dev/null
@@ -1,656 +0,0 @@
-/*
- * /+\
- * +\ Copyright 1993-2002 Christopher Seiwald and Perforce Software, Inc.
- * \+/
- *
- * This file is part of jam.
- *
- * License is hereby granted to use this software and distribute it freely, as
- * long as this copyright notice is retained and modifications are clearly
- * marked.
- *
- * ALL WARRANTIES ARE HEREBY DISCLAIMED.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * jam.c - make redux
- *
- * See Jam.html for usage information.
- *
- * These comments document the code.
- *
- * The top half of the code is structured such:
- *
- * jam
- * / | \
- * +---+ | \
- * / | \
- * jamgram option \
- * / | \ \
- * / | \ \
- * / | \ |
- * scan | compile make
- * | | / | \ / | \
- * | | / | \ / | \
- * | | / | \ / | \
- * jambase parse | rules search make1
- * | | | \
- * | | | \
- * | | | \
- * builtins timestamp command execute
- * |
- * |
- * |
- * filesys
- *
- *
- * The support routines are called by all of the above, but themselves are
- * layered thus:
- *
- * variable|expand
- * / | |
- * / | |
- * / | |
- * lists | pathsys
- * \ |
- * \ hash
- * \ |
- * \ |
- * \ |
- * \ |
- * \ |
- * object
- *
- * Roughly, the modules are:
- *
- * builtins.c - jam's built-in rules
- * command.c - maintain lists of commands
- * compile.c - compile parsed jam statements
- * exec*.c - execute a shell script on a specific OS
- * file*.c - scan directories and archives on a specific OS
- * hash.c - simple in-memory hashing routines
- * hdrmacro.c - handle header file parsing for filename macro definitions
- * headers.c - handle #includes in source files
- * jambase.c - compilable copy of Jambase
- * jamgram.y - jam grammar
- * lists.c - maintain lists of strings
- * make.c - bring a target up to date, once rules are in place
- * make1.c - execute command to bring targets up to date
- * object.c - string manipulation routines
- * option.c - command line option processing
- * parse.c - make and destroy parse trees as driven by the parser
- * path*.c - manipulate file names on a specific OS
- * hash.c - simple in-memory hashing routines
- * regexp.c - Henry Spencer's regexp
- * rules.c - access to RULEs, TARGETs, and ACTIONs
- * scan.c - the jam yacc scanner
- * search.c - find a target along $(SEARCH) or $(LOCATE)
- * timestamp.c - get the timestamp of a file or archive member
- * variable.c - handle jam multi-element variables
- */
-
-
-#include "jam.h"
-#include "patchlevel.h"
-
-#include "builtins.h"
-#include "class.h"
-#include "compile.h"
-#include "constants.h"
-#include "filesys.h"
-#include "function.h"
-#include "hcache.h"
-#include "lists.h"
-#include "make.h"
-#include "object.h"
-#include "option.h"
-#include "output.h"
-#include "parse.h"
-#include "cwd.h"
-#include "rules.h"
-#include "scan.h"
-#include "search.h"
-#include "strings.h"
-#include "timestamp.h"
-#include "variable.h"
-
-/* Macintosh is "special" */
-#ifdef OS_MAC
-# include <QuickDraw.h>
-#endif
-
-/* And UNIX for this. */
-#ifdef unix
-# include <sys/utsname.h>
-# include <signal.h>
-#endif
-
-struct globs globs =
-{
- 0, /* noexec */
- 1, /* jobs */
- 0, /* quitquick */
- 0, /* newestfirst */
- 0, /* pipes action stdout and stderr merged to action output */
-#ifdef OS_MAC
- { 0, 0 }, /* debug - suppress tracing output */
-#else
- { 0, 1 }, /* debug ... */
-#endif
- 0, /* output commands, not run them */
- 0, /* action timeout */
- 0 /* maximum buffer size zero is all output */
-};
-
-/* Symbols to be defined as true for use in Jambase. */
-static char * othersyms[] = { OSMAJOR, OSMINOR, OSPLAT, JAMVERSYM, 0 };
-
-
-/* Known for sure:
- * mac needs arg_enviro
- * OS2 needs extern environ
- */
-
-#ifdef OS_MAC
-# define use_environ arg_environ
-# ifdef MPW
- QDGlobals qd;
-# endif
-#endif
-
-/* on Win32-LCC */
-#if defined( OS_NT ) && defined( __LCC__ )
-# define use_environ _environ
-#endif
-
-#if defined( __MWERKS__)
-# define use_environ _environ
- extern char * * _environ;
-#endif
-
-#ifndef use_environ
-# define use_environ environ
-# if !defined( __WATCOM__ ) && !defined( OS_OS2 ) && !defined( OS_NT )
- extern char **environ;
-# endif
-#endif
-
-#if YYDEBUG != 0
- extern int yydebug;
-#endif
-
-#ifndef NDEBUG
-static void run_unit_tests()
-{
-# if defined( USE_EXECNT )
- extern void execnt_unit_test();
- execnt_unit_test();
-# endif
- string_unit_test();
-}
-#endif
-
-int anyhow = 0;
-
-#ifdef HAVE_PYTHON
- extern PyObject * bjam_call ( PyObject * self, PyObject * args );
- extern PyObject * bjam_import_rule ( PyObject * self, PyObject * args );
- extern PyObject * bjam_define_action( PyObject * self, PyObject * args );
- extern PyObject * bjam_variable ( PyObject * self, PyObject * args );
- extern PyObject * bjam_backtrace ( PyObject * self, PyObject * args );
- extern PyObject * bjam_caller ( PyObject * self, PyObject * args );
-#endif
-
-void regex_done();
-
-char const * saved_argv0;
-
-int main( int argc, char * * argv, char * * arg_environ )
-{
- int n;
- char * s;
- struct bjam_option optv[ N_OPTS ];
- char const * all = "all";
- int status;
- int arg_c = argc;
- char * * arg_v = argv;
- char const * progname = argv[ 0 ];
- module_t * environ_module;
-
- saved_argv0 = argv[ 0 ];
-
- BJAM_MEM_INIT();
-
-#ifdef OS_MAC
- InitGraf( &qd.thePort );
-#endif
-
- --argc;
- ++argv;
-
- if ( getoptions( argc, argv, "-:l:m:d:j:p:f:gs:t:ano:qv", optv ) < 0 )
- {
- printf( "\nusage: %s [ options ] targets...\n\n", progname );
-
- printf( "-a Build all targets, even if they are current.\n" );
- printf( "-dx Set the debug level to x (0-9).\n" );
- printf( "-fx Read x instead of Jambase.\n" );
- /* printf( "-g Build from newest sources first.\n" ); */
- printf( "-jx Run up to x shell commands concurrently.\n" );
- printf( "-lx Limit actions to x number of seconds after which they are stopped.\n" );
- printf( "-mx Maximum target output saved (kb), default is to save all output.\n" );
- printf( "-n Don't actually execute the updating actions.\n" );
- printf( "-ox Write the updating actions to file x.\n" );
- printf( "-px x=0, pipes action stdout and stderr merged into action output.\n" );
- printf( "-q Quit quickly as soon as a target fails.\n" );
- printf( "-sx=y Set variable x=y, overriding environment.\n" );
- printf( "-tx Rebuild x, even if it is up-to-date.\n" );
- printf( "-v Print the version of jam and exit.\n" );
- printf( "--x Option is ignored.\n\n" );
-
- exit( EXITBAD );
- }
-
- /* Version info. */
- if ( ( s = getoptval( optv, 'v', 0 ) ) )
- {
- printf( "Boost.Jam Version %s. %s.\n", VERSION, OSMINOR );
- printf( " Copyright 1993-2002 Christopher Seiwald and Perforce "
- "Software, Inc.\n" );
- printf( " Copyright 2001 David Turner.\n" );
- printf( " Copyright 2001-2004 David Abrahams.\n" );
- printf( " Copyright 2002-2008 Rene Rivera.\n" );
- printf( " Copyright 2003-2008 Vladimir Prus.\n" );
- return EXITOK;
- }
-
- /* Pick up interesting options. */
- if ( ( s = getoptval( optv, 'n', 0 ) ) )
- {
- ++globs.noexec;
- globs.debug[ 2 ] = 1;
- }
-
- if ( ( s = getoptval( optv, 'p', 0 ) ) )
- {
- /* Undocumented -p3 (acts like both -p1 -p2) means separate pipe action
- * stdout and stderr.
- */
- globs.pipe_action = atoi( s );
- if ( globs.pipe_action < 0 || 3 < globs.pipe_action )
- {
- printf( "Invalid pipe descriptor '%d', valid values are -p[0..3]."
- "\n", globs.pipe_action );
- exit( EXITBAD );
- }
- }
-
- if ( ( s = getoptval( optv, 'q', 0 ) ) )
- globs.quitquick = 1;
-
- if ( ( s = getoptval( optv, 'a', 0 ) ) )
- anyhow++;
-
- if ( ( s = getoptval( optv, 'j', 0 ) ) )
- {
- globs.jobs = atoi( s );
- if ( globs.jobs < 1 || globs.jobs > MAXJOBS )
- {
- printf( "Invalid value for the '-j' option, valid values are 1 "
- "through %d.\n", MAXJOBS );
- exit( EXITBAD );
- }
- }
-
- if ( ( s = getoptval( optv, 'g', 0 ) ) )
- globs.newestfirst = 1;
-
- if ( ( s = getoptval( optv, 'l', 0 ) ) )
- globs.timeout = atoi( s );
-
- if ( ( s = getoptval( optv, 'm', 0 ) ) )
- globs.max_buf = atoi( s ) * 1024; /* convert to kb */
-
- /* Turn on/off debugging */
- for ( n = 0; ( s = getoptval( optv, 'd', n ) ); ++n )
- {
- int i;
-
- /* First -d, turn off defaults. */
- if ( !n )
- for ( i = 0; i < DEBUG_MAX; ++i )
- globs.debug[i] = 0;
-
- i = atoi( s );
-
- if ( ( i < 0 ) || ( i >= DEBUG_MAX ) )
- {
- printf( "Invalid debug level '%s'.\n", s );
- continue;
- }
-
- /* n turns on levels 1-n. */
- /* +n turns on level n. */
- if ( *s == '+' )
- globs.debug[ i ] = 1;
- else while ( i )
- globs.debug[ i-- ] = 1;
- }
-
- constants_init();
- cwd_init();
-
- {
- PROFILE_ENTER( MAIN );
-
-#ifdef HAVE_PYTHON
- {
- PROFILE_ENTER( MAIN_PYTHON );
- Py_Initialize();
- {
- static PyMethodDef BjamMethods[] = {
- {"call", bjam_call, METH_VARARGS,
- "Call the specified bjam rule."},
- {"import_rule", bjam_import_rule, METH_VARARGS,
- "Imports Python callable to bjam."},
- {"define_action", bjam_define_action, METH_VARARGS,
- "Defines a command line action."},
- {"variable", bjam_variable, METH_VARARGS,
- "Obtains a variable from bjam's global module."},
- {"backtrace", bjam_backtrace, METH_VARARGS,
- "Returns bjam backtrace from the last call into Python."},
- {"caller", bjam_caller, METH_VARARGS,
- "Returns the module from which the last call into Python is made."},
- {NULL, NULL, 0, NULL}
- };
-
- Py_InitModule( "bjam", BjamMethods );
- }
- PROFILE_EXIT( MAIN_PYTHON );
- }
-#endif
-
-#ifndef NDEBUG
- run_unit_tests();
-#endif
-#if YYDEBUG != 0
- if ( DEBUG_PARSE )
- yydebug = 1;
-#endif
-
- /* Set JAMDATE. */
- {
- timestamp current;
- timestamp_current( ¤t );
- var_set( root_module(), constant_JAMDATE, list_new( outf_time(
- ¤t ) ), VAR_SET );
- }
-
- /* Set JAM_VERSION. */
- var_set( root_module(), constant_JAM_VERSION,
- list_push_back( list_push_back( list_new(
- object_new( VERSION_MAJOR_SYM ) ),
- object_new( VERSION_MINOR_SYM ) ),
- object_new( VERSION_PATCH_SYM ) ),
- VAR_SET );
-
- /* Set JAMUNAME. */
-#ifdef unix
- {
- struct utsname u;
-
- if ( uname( &u ) >= 0 )
- {
- var_set( root_module(), constant_JAMUNAME,
- list_push_back(
- list_push_back(
- list_push_back(
- list_push_back(
- list_new(
- object_new( u.sysname ) ),
- object_new( u.nodename ) ),
- object_new( u.release ) ),
- object_new( u.version ) ),
- object_new( u.machine ) ), VAR_SET );
- }
- }
-#endif /* unix */
-
- /* Set JAM_TIMESTAMP_RESOLUTION. */
- {
- timestamp fmt_resolution[ 1 ];
- file_supported_fmt_resolution( fmt_resolution );
- var_set( root_module(), constant_JAM_TIMESTAMP_RESOLUTION, list_new(
- object_new( timestamp_timestr( fmt_resolution ) ) ), VAR_SET );
- }
-
- /* Load up environment variables. */
-
- /* First into the global module, with splitting, for backward
- * compatibility.
- */
- var_defines( root_module(), use_environ, 1 );
-
- environ_module = bindmodule( constant_ENVIRON );
- /* Then into .ENVIRON, without splitting. */
- var_defines( environ_module, use_environ, 0 );
-
- /*
- * Jam defined variables OS & OSPLAT. We load them after environment, so
- * that setting OS in environment does not change Jam's notion of the
- * current platform.
- */
- var_defines( root_module(), othersyms, 1 );
-
- /* Load up variables set on command line. */
- for ( n = 0; ( s = getoptval( optv, 's', n ) ); ++n )
- {
- char * symv[ 2 ];
- symv[ 0 ] = s;
- symv[ 1 ] = 0;
- var_defines( root_module(), symv, 1 );
- var_defines( environ_module, symv, 0 );
- }
-
- /* Set the ARGV to reflect the complete list of arguments of invocation.
- */
- for ( n = 0; n < arg_c; ++n )
- var_set( root_module(), constant_ARGV, list_new( object_new(
- arg_v[ n ] ) ), VAR_APPEND );
-
- /* Initialize built-in rules. */
- load_builtins();
-
- /* Add the targets in the command line to the update list. */
- for ( n = 1; n < arg_c; ++n )
- {
- if ( arg_v[ n ][ 0 ] == '-' )
- {
- char * f = "-:l:d:j:f:gs:t:ano:qv";
- for ( ; *f; ++f ) if ( *f == arg_v[ n ][ 1 ] ) break;
- if ( ( f[ 1 ] == ':' ) && ( arg_v[ n ][ 2 ] == '\0' ) ) ++n;
- }
- else
- {
- OBJECT * const target = object_new( arg_v[ n ] );
- mark_target_for_updating( target );
- object_free( target );
- }
- }
-
- if ( list_empty( targets_to_update() ) )
- mark_target_for_updating( constant_all );
-
- /* Parse ruleset. */
- {
- FRAME frame[ 1 ];
- frame_init( frame );
- for ( n = 0; ( s = getoptval( optv, 'f', n ) ); ++n )
- {
- OBJECT * const filename = object_new( s );
- parse_file( filename, frame );
- object_free( filename );
- }
-
- if ( !n )
- parse_file( constant_plus, frame );
- }
-
- status = yyanyerrors();
-
- /* Manually touch -t targets. */
- for ( n = 0; ( s = getoptval( optv, 't', n ) ); ++n )
- {
- OBJECT * const target = object_new( s );
- touch_target( target );
- object_free( target );
- }
-
- /* If an output file is specified, set globs.cmdout to that. */
- if ( ( s = getoptval( optv, 'o', 0 ) ) )
- {
- if ( !( globs.cmdout = fopen( s, "w" ) ) )
- {
- printf( "Failed to write to '%s'\n", s );
- exit( EXITBAD );
- }
- ++globs.noexec;
- }
-
- /* The build system may set the PARALLELISM variable to override -j
- * options.
- */
- {
- LIST * const p = var_get( root_module(), constant_PARALLELISM );
- if ( !list_empty( p ) )
- {
- int const j = atoi( object_str( list_front( p ) ) );
- if ( j < 1 || j > MAXJOBS )
- printf( "Invalid value of PARALLELISM: %s. Valid values "
- "are 1 through %d.\n", object_str( list_front( p ) ),
- MAXJOBS );
- else
- globs.jobs = j;
- }
- }
-
- /* KEEP_GOING overrides -q option. */
- {
- LIST * const p = var_get( root_module(), constant_KEEP_GOING );
- if ( !list_empty( p ) )
- globs.quitquick = atoi( object_str( list_front( p ) ) ) ? 0 : 1;
- }
-
- /* Now make target. */
- {
- PROFILE_ENTER( MAIN_MAKE );
- LIST * const targets = targets_to_update();
- if ( !list_empty( targets ) )
- status |= make( targets, anyhow );
- else
- status = last_update_now_status;
- PROFILE_EXIT( MAIN_MAKE );
- }
-
- PROFILE_EXIT( MAIN );
- }
-
- if ( DEBUG_PROFILE )
- profile_dump();
-
-
-#ifdef OPT_HEADER_CACHE_EXT
- hcache_done();
-#endif
-
- clear_targets_to_update();
-
- /* Widely scattered cleanup. */
- property_set_done();
- file_done();
- rules_done();
- timestamp_done();
- search_done();
- class_done();
- modules_done();
- regex_done();
- cwd_done();
- path_done();
- function_done();
- list_done();
- constants_done();
- object_done();
-
- /* Close cmdout. */
- if ( globs.cmdout )
- fclose( globs.cmdout );
-
-#ifdef HAVE_PYTHON
- Py_Finalize();
-#endif
-
- BJAM_MEM_CLOSE();
-
- return status ? EXITBAD : EXITOK;
-}
-
-
-/*
- * executable_path()
- */
-
-#if defined(_WIN32)
-# define WIN32_LEAN_AND_MEAN
-# include <windows.h>
-char * executable_path( char const * argv0 )
-{
- char buf[ 1024 ];
- DWORD const ret = GetModuleFileName( NULL, buf, sizeof( buf ) );
- return ( !ret || ret == sizeof( buf ) ) ? NULL : strdup( buf );
-}
-#elif defined(__APPLE__) /* Not tested */
-# include <mach-o/dyld.h>
-char *executable_path( char const * argv0 )
-{
- char buf[ 1024 ];
- uint32_t size = sizeof( buf );
- return _NSGetExecutablePath( buf, &size ) ? NULL : strdup( buf );
-}
-#elif defined(sun) || defined(__sun) /* Not tested */
-# include <stdlib.h>
-char * executable_path( char const * argv0 )
-{
- return strdup( getexecname() );
-}
-#elif defined(__FreeBSD__)
-# include <sys/sysctl.h>
-char * executable_path( char const * argv0 )
-{
- int mib[ 4 ] = { CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1 };
- char buf[ 1024 ];
- size_t size = sizeof( buf );
- sysctl( mib, 4, buf, &size, NULL, 0 );
- return ( !size || size == sizeof( buf ) ) ? NULL : strndup( buf, size );
-}
-#elif defined(__linux__)
-# include <unistd.h>
-char * executable_path( char const * argv0 )
-{
- char buf[ 1024 ];
- ssize_t const ret = readlink( "/proc/self/exe", buf, sizeof( buf ) );
- return ( !ret || ret == sizeof( buf ) ) ? NULL : strndup( buf, ret );
-}
-#else
-char * executable_path( char const * argv0 )
-{
- /* If argv0 is an absolute path, assume it is the right absolute path. */
- return argv0[ 0 ] == '/' ? strdup( argv0 ) : NULL;
-}
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/jam.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/jam.h b/ext/kenlm/jam-files/engine/jam.h
deleted file mode 100644
index 86ad0e8..0000000
--- a/ext/kenlm/jam-files/engine/jam.h
+++ /dev/null
@@ -1,475 +0,0 @@
-/*
- * Copyright 1993, 1995 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * jam.h - includes and globals for jam
- */
-
-#ifndef JAM_H_VP_2003_08_01
-#define JAM_H_VP_2003_08_01
-
-#ifdef HAVE_PYTHON
-#include <Python.h>
-#endif
-
-/* Assume popen support is available unless known otherwise. */
-#define HAVE_POPEN 1
-
-/*
- * Windows NT
- */
-
-#ifdef NT
-
-#include <ctype.h>
-#include <fcntl.h>
-#include <malloc.h>
-#ifndef __MWERKS__
- #include <memory.h>
-#endif
-#include <stdio.h>
-#include <stdlib.h>
-#include <signal.h>
-#include <string.h>
-#include <time.h>
-
-#define OSMAJOR "NT=true"
-#define OSMINOR "OS=NT"
-#define OS_NT
-#define SPLITPATH ';'
-#define MAXLINE (undefined__see_execnt_c) /* max chars per command line */
-#define USE_EXECNT
-#define PATH_DELIM '\\'
-
-/* AS400 cross-compile from NT. */
-
-#ifdef AS400
- #undef OSMINOR
- #undef OSMAJOR
- #define OSMAJOR "AS400=true"
- #define OSMINOR "OS=AS400"
- #define OS_AS400
-#endif
-
-/* Metrowerks Standard Library on Windows. */
-
-#ifdef __MSL__
- #undef HAVE_POPEN
-#endif
-
-#endif /* #ifdef NT */
-
-
-/*
- * Windows MingW32
- */
-
-#ifdef MINGW
-
-#include <fcntl.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <ctype.h>
-#include <malloc.h>
-#include <memory.h>
-#include <signal.h>
-#include <string.h>
-#include <time.h>
-
-#define OSMAJOR "MINGW=true"
-#define OSMINOR "OS=MINGW"
-#define OS_NT
-#define SPLITPATH ';'
-#define MAXLINE 996 /* max chars per command line */
-#define USE_EXECUNIX
-#define PATH_DELIM '\\'
-
-#endif /* #ifdef MINGW */
-
-
-/*
- * God fearing UNIX.
- */
-
-#ifndef OSMINOR
-
-#define OSMAJOR "UNIX=true"
-#define USE_EXECUNIX
-#define USE_FILEUNIX
-#define PATH_DELIM '/'
-
-#ifdef _AIX
- #define unix
- #define MAXLINE 23552 /* 24k - 1k, max chars per command line */
- #define OSMINOR "OS=AIX"
- #define OS_AIX
- #define NO_VFORK
-#endif
-#ifdef AMIGA
- #define OSMINOR "OS=AMIGA"
- #define OS_AMIGA
-#endif
-#ifdef __BEOS__
- #define unix
- #define OSMINOR "OS=BEOS"
- #define OS_BEOS
- #define NO_VFORK
-#endif
-#ifdef __bsdi__
- #define OSMINOR "OS=BSDI"
- #define OS_BSDI
-#endif
-#if defined (COHERENT) && defined (_I386)
- #define OSMINOR "OS=COHERENT"
- #define OS_COHERENT
- #define NO_VFORK
-#endif
-#if defined(__cygwin__) || defined(__CYGWIN__)
- #define OSMINOR "OS=CYGWIN"
- #define OS_CYGWIN
-#endif
-#if defined(__FreeBSD__) && !defined(__DragonFly__)
- #define OSMINOR "OS=FREEBSD"
- #define OS_FREEBSD
-#endif
-#ifdef __DragonFly__
- #define OSMINOR "OS=DRAGONFLYBSD"
- #define OS_DRAGONFLYBSD
-#endif
-#ifdef __DGUX__
- #define OSMINOR "OS=DGUX"
- #define OS_DGUX
-#endif
-#ifdef __hpux
- #define OSMINOR "OS=HPUX"
- #define OS_HPUX
-#endif
-#ifdef __OPENNT
- #define unix
- #define OSMINOR "OS=INTERIX"
- #define OS_INTERIX
- #define NO_VFORK
-#endif
-#ifdef __sgi
- #define OSMINOR "OS=IRIX"
- #define OS_IRIX
- #define NO_VFORK
-#endif
-#ifdef __ISC
- #define OSMINOR "OS=ISC"
- #define OS_ISC
- #define NO_VFORK
-#endif
-#ifdef linux
- #define OSMINOR "OS=LINUX"
- #define OS_LINUX
-#endif
-#ifdef __Lynx__
- #define OSMINOR "OS=LYNX"
- #define OS_LYNX
- #define NO_VFORK
- #define unix
-#endif
-#ifdef __MACHTEN__
- #define OSMINOR "OS=MACHTEN"
- #define OS_MACHTEN
-#endif
-#ifdef mpeix
- #define unix
- #define OSMINOR "OS=MPEIX"
- #define OS_MPEIX
- #define NO_VFORK
-#endif
-#ifdef __MVS__
- #define unix
- #define OSMINOR "OS=MVS"
- #define OS_MVS
-#endif
-#ifdef _ATT4
- #define OSMINOR "OS=NCR"
- #define OS_NCR
-#endif
-#ifdef __NetBSD__
- #define unix
- #define OSMINOR "OS=NETBSD"
- #define OS_NETBSD
- #define NO_VFORK
-#endif
-#ifdef __QNX__
- #define unix
- #ifdef __QNXNTO__
- #define OSMINOR "OS=QNXNTO"
- #define OS_QNXNTO
- #else
- #define OSMINOR "OS=QNX"
- #define OS_QNX
- #define NO_VFORK
- #define MAXLINE 996 /* max chars per command line */
- #endif
-#endif
-#ifdef NeXT
- #ifdef __APPLE__
- #define OSMINOR "OS=RHAPSODY"
- #define OS_RHAPSODY
- #else
- #define OSMINOR "OS=NEXT"
- #define OS_NEXT
- #endif
-#endif
-#ifdef __APPLE__
- #define unix
- #define OSMINOR "OS=MACOSX"
- #define OS_MACOSX
-#endif
-#ifdef __osf__
- #ifndef unix
- #define unix
- #endif
- #define OSMINOR "OS=OSF"
- #define OS_OSF
-#endif
-#ifdef _SEQUENT_
- #define OSMINOR "OS=PTX"
- #define OS_PTX
-#endif
-#ifdef M_XENIX
- #define OSMINOR "OS=SCO"
- #define OS_SCO
- #define NO_VFORK
-#endif
-#ifdef sinix
- #define unix
- #define OSMINOR "OS=SINIX"
- #define OS_SINIX
-#endif
-#ifdef sun
- #if defined(__svr4__) || defined(__SVR4)
- #define OSMINOR "OS=SOLARIS"
- #define OS_SOLARIS
- #else
- #define OSMINOR "OS=SUNOS"
- #define OS_SUNOS
- #endif
-#endif
-#ifdef ultrix
- #define OSMINOR "OS=ULTRIX"
- #define OS_ULTRIX
-#endif
-#ifdef _UNICOS
- #define OSMINOR "OS=UNICOS"
- #define OS_UNICOS
-#endif
-#if defined(__USLC__) && !defined(M_XENIX)
- #define OSMINOR "OS=UNIXWARE"
- #define OS_UNIXWARE
-#endif
-#ifdef __OpenBSD__
- #define OSMINOR "OS=OPENBSD"
- #define OS_OPENBSD
- #define unix
-#endif
-#if defined (__FreeBSD_kernel__) && !defined(__FreeBSD__)
- #define OSMINOR "OS=KFREEBSD"
- #define OS_KFREEBSD
-#endif
-#ifndef OSMINOR
- #define OSMINOR "OS=UNKNOWN"
-#endif
-
-/* All the UNIX includes */
-
-#include <sys/types.h>
-
-#ifndef OS_MPEIX
- #include <sys/file.h>
-#endif
-
-#include <fcntl.h>
-#include <stdio.h>
-#include <ctype.h>
-#include <signal.h>
-#include <string.h>
-#include <time.h>
-#include <unistd.h>
-
-#ifndef OS_QNX
- #include <memory.h>
-#endif
-
-#ifndef OS_ULTRIX
- #include <stdlib.h>
-#endif
-
-#if !defined( OS_BSDI ) && \
- !defined( OS_FREEBSD ) && \
- !defined( OS_DRAGONFLYBSD ) && \
- !defined( OS_NEXT ) && \
- !defined( OS_MACHTEN ) && \
- !defined( OS_MACOSX ) && \
- !defined( OS_RHAPSODY ) && \
- !defined( OS_MVS ) && \
- !defined( OS_OPENBSD )
- #include <malloc.h>
-#endif
-
-#endif /* #ifndef OSMINOR */
-
-
-/*
- * OSPLAT definitions - suppressed when it is a one-of-a-kind.
- */
-
-#if defined( _M_PPC ) || \
- defined( PPC ) || \
- defined( ppc ) || \
- defined( __powerpc__ ) || \
- defined( __ppc__ )
- #define OSPLAT "OSPLAT=PPC"
-#endif
-
-#if defined( _ALPHA_ ) || \
- defined( __alpha__ )
- #define OSPLAT "OSPLAT=AXP"
-#endif
-
-#if defined( _i386_ ) || \
- defined( __i386__ ) || \
- defined( __i386 ) || \
- defined( _M_IX86 )
- #define OSPLAT "OSPLAT=X86"
-#endif
-
-#if defined( __ia64__ ) || \
- defined( __IA64__ ) || \
- defined( __ia64 )
- #define OSPLAT "OSPLAT=IA64"
-#endif
-
-#if defined( __x86_64__ ) || \
- defined( __amd64__ ) || \
- defined( _M_AMD64 )
- #define OSPLAT "OSPLAT=X86_64"
-#endif
-
-#if defined( __sparc__ ) || \
- defined( __sparc )
- #define OSPLAT "OSPLAT=SPARC"
-#endif
-
-#ifdef __mips__
- #define OSPLAT "OSPLAT=MIPS"
-#endif
-
-#ifdef __arm__
- #define OSPLAT "OSPLAT=ARM"
-#endif
-
-#ifdef __s390__
- #define OSPLAT "OSPLAT=390"
-#endif
-
-#ifdef __hppa
- #define OSPLAT "OSPLAT=PARISC"
-#endif
-
-#ifndef OSPLAT
- #define OSPLAT ""
-#endif
-
-
-/*
- * Jam implementation misc.
- */
-
-#ifndef MAXLINE
- #define MAXLINE 102400 /* max chars per command line */
-#endif
-
-#ifndef EXITOK
- #define EXITOK 0
- #define EXITBAD 1
-#endif
-
-#ifndef SPLITPATH
- #define SPLITPATH ':'
-#endif
-
-/* You probably do not need to muck with these. */
-
-#define MAXSYM 1024 /* longest symbol in the environment */
-#define MAXJPATH 1024 /* longest filename */
-
-#define MAXJOBS 64 /* internally enforced -j limit */
-#define MAXARGC 32 /* words in $(JAMSHELL) */
-
-/* Jam private definitions below. */
-
-#define DEBUG_MAX 14
-
-
-struct globs
-{
- int noexec;
- int jobs;
- int quitquick;
- int newestfirst; /* build newest sources first */
- int pipe_action;
- char debug[ DEBUG_MAX ];
- FILE * cmdout; /* print cmds, not run them */
- long timeout; /* number of seconds to limit actions to,
- * default 0 for no limit.
- */
- int dart; /* output build and test results formatted for
- * Dart
- */
- int max_buf; /* maximum amount of output saved from target
- * (kb)
- */
-};
-
-extern struct globs globs;
-
-#define DEBUG_MAKE ( globs.debug[ 1 ] ) /* show actions when executed */
-#define DEBUG_MAKEQ ( globs.debug[ 2 ] ) /* show even quiet actions */
-#define DEBUG_EXEC ( globs.debug[ 2 ] ) /* show text of actons */
-#define DEBUG_MAKEPROG ( globs.debug[ 3 ] ) /* show make0 progress */
-#define DEBUG_BIND ( globs.debug[ 3 ] ) /* show when files bound */
-
-#define DEBUG_EXECCMD ( globs.debug[ 4 ] ) /* show execcmds()'s work */
-
-#define DEBUG_COMPILE ( globs.debug[ 5 ] ) /* show rule invocations */
-
-#define DEBUG_HEADER ( globs.debug[ 6 ] ) /* show result of header scan */
-#define DEBUG_BINDSCAN ( globs.debug[ 6 ] ) /* show result of dir scan */
-#define DEBUG_SEARCH ( globs.debug[ 6 ] ) /* show binding attempts */
-
-#define DEBUG_VARSET ( globs.debug[ 7 ] ) /* show variable settings */
-#define DEBUG_VARGET ( globs.debug[ 8 ] ) /* show variable fetches */
-#define DEBUG_VAREXP ( globs.debug[ 8 ] ) /* show variable expansions */
-#define DEBUG_IF ( globs.debug[ 8 ] ) /* show 'if' calculations */
-#define DEBUG_LISTS ( globs.debug[ 9 ] ) /* show list manipulation */
-#define DEBUG_SCAN ( globs.debug[ 9 ] ) /* show scanner tokens */
-#define DEBUG_MEM ( globs.debug[ 9 ] ) /* show memory use */
-
-#define DEBUG_PROFILE ( globs.debug[ 10 ] ) /* dump rule execution times */
-#define DEBUG_PARSE ( globs.debug[ 11 ] ) /* debug parsing */
-#define DEBUG_GRAPH ( globs.debug[ 12 ] ) /* debug dependencies */
-#define DEBUG_FATE ( globs.debug[ 13 ] ) /* show fate changes in make0() */
-
-/* Everyone gets the memory definitions. */
-#include "mem.h"
-
-/* They also get the profile functions. */
-#include "debug.h"
-
-#endif
[14/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/builder/initial_probabilities.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/builder/initial_probabilities.cc b/ext/kenlm/lm/builder/initial_probabilities.cc
deleted file mode 100644
index 5b8d86d..0000000
--- a/ext/kenlm/lm/builder/initial_probabilities.cc
+++ /dev/null
@@ -1,306 +0,0 @@
-#include "lm/builder/initial_probabilities.hh"
-
-#include "lm/builder/discount.hh"
-#include "lm/builder/hash_gamma.hh"
-#include "lm/builder/payload.hh"
-#include "lm/common/special.hh"
-#include "lm/common/ngram_stream.hh"
-#include "util/murmur_hash.hh"
-#include "util/file.hh"
-#include "util/stream/chain.hh"
-#include "util/stream/io.hh"
-#include "util/stream/stream.hh"
-
-#include <vector>
-
-namespace lm { namespace builder {
-
-namespace {
-struct BufferEntry {
- // Gamma from page 20 of Chen and Goodman.
- float gamma;
- // \sum_w a(c w) for all w.
- float denominator;
-};
-
-struct HashBufferEntry : public BufferEntry {
- // Hash value of ngram. Used to join contexts with backoffs.
- uint64_t hash_value;
-};
-
-// Reads all entries in order like NGramStream does.
-// But deletes any entries that have CutoffCount below or equal to pruning
-// threshold.
-class PruneNGramStream {
- public:
- PruneNGramStream(const util::stream::ChainPosition &position, const SpecialVocab &specials) :
- current_(NULL, NGram<BuildingPayload>::OrderFromSize(position.GetChain().EntrySize())),
- dest_(NULL, NGram<BuildingPayload>::OrderFromSize(position.GetChain().EntrySize())),
- currentCount_(0),
- block_(position),
- specials_(specials)
- {
- StartBlock();
- }
-
- NGram<BuildingPayload> &operator*() { return current_; }
- NGram<BuildingPayload> *operator->() { return ¤t_; }
-
- operator bool() const {
- return block_;
- }
-
- PruneNGramStream &operator++() {
- assert(block_);
- if(UTIL_UNLIKELY(current_.Order() == 1 && specials_.IsSpecial(*current_.begin())))
- dest_.NextInMemory();
- else if(currentCount_ > 0) {
- if(dest_.Base() < current_.Base()) {
- memcpy(dest_.Base(), current_.Base(), current_.TotalSize());
- }
- dest_.NextInMemory();
- }
-
- current_.NextInMemory();
-
- uint8_t *block_base = static_cast<uint8_t*>(block_->Get());
- if (current_.Base() == block_base + block_->ValidSize()) {
- block_->SetValidSize(dest_.Base() - block_base);
- ++block_;
- StartBlock();
- if (block_) {
- currentCount_ = current_.Value().CutoffCount();
- }
- } else {
- currentCount_ = current_.Value().CutoffCount();
- }
-
- return *this;
- }
-
- private:
- void StartBlock() {
- for (; ; ++block_) {
- if (!block_) return;
- if (block_->ValidSize()) break;
- }
- current_.ReBase(block_->Get());
- currentCount_ = current_.Value().CutoffCount();
-
- dest_.ReBase(block_->Get());
- }
-
- NGram<BuildingPayload> current_; // input iterator
- NGram<BuildingPayload> dest_; // output iterator
-
- uint64_t currentCount_;
-
- util::stream::Link block_;
-
- const SpecialVocab specials_;
-};
-
-// Extract an array of HashedGamma from an array of BufferEntry.
-class OnlyGamma {
- public:
- explicit OnlyGamma(bool pruning) : pruning_(pruning) {}
-
- void Run(const util::stream::ChainPosition &position) {
- for (util::stream::Link block_it(position); block_it; ++block_it) {
- if(pruning_) {
- const HashBufferEntry *in = static_cast<const HashBufferEntry*>(block_it->Get());
- const HashBufferEntry *end = static_cast<const HashBufferEntry*>(block_it->ValidEnd());
-
- // Just make it point to the beginning of the stream so it can be overwritten
- // With HashGamma values. Do not attempt to interpret the values until set below.
- HashGamma *out = static_cast<HashGamma*>(block_it->Get());
- for (; in < end; out += 1, in += 1) {
- // buffering, otherwise might overwrite values too early
- float gamma_buf = in->gamma;
- uint64_t hash_buf = in->hash_value;
-
- out->gamma = gamma_buf;
- out->hash_value = hash_buf;
- }
- block_it->SetValidSize((block_it->ValidSize() * sizeof(HashGamma)) / sizeof(HashBufferEntry));
- }
- else {
- float *out = static_cast<float*>(block_it->Get());
- const float *in = out;
- const float *end = static_cast<const float*>(block_it->ValidEnd());
- for (out += 1, in += 2; in < end; out += 1, in += 2) {
- *out = *in;
- }
- block_it->SetValidSize(block_it->ValidSize() / 2);
- }
- }
- }
-
- private:
- bool pruning_;
-};
-
-class AddRight {
- public:
- AddRight(const Discount &discount, const util::stream::ChainPosition &input, bool pruning)
- : discount_(discount), input_(input), pruning_(pruning) {}
-
- void Run(const util::stream::ChainPosition &output) {
- NGramStream<BuildingPayload> in(input_);
- util::stream::Stream out(output);
-
- std::vector<WordIndex> previous(in->Order() - 1);
- // Silly windows requires this workaround to just get an invalid pointer when empty.
- void *const previous_raw = previous.empty() ? NULL : static_cast<void*>(&previous[0]);
- const std::size_t size = sizeof(WordIndex) * previous.size();
-
- for(; in; ++out) {
- memcpy(previous_raw, in->begin(), size);
- uint64_t denominator = 0;
- uint64_t normalizer = 0;
-
- uint64_t counts[4];
- memset(counts, 0, sizeof(counts));
- do {
- denominator += in->Value().UnmarkedCount();
-
- // Collect unused probability mass from pruning.
- // Becomes 0 for unpruned ngrams.
- normalizer += in->Value().UnmarkedCount() - in->Value().CutoffCount();
-
- // Chen&Goodman do not mention counting based on cutoffs, but
- // backoff becomes larger than 1 otherwise, so probably needs
- // to count cutoffs. Counts normally without pruning.
- if(in->Value().CutoffCount() > 0)
- ++counts[std::min(in->Value().CutoffCount(), static_cast<uint64_t>(3))];
-
- } while (++in && !memcmp(previous_raw, in->begin(), size));
-
- BufferEntry &entry = *reinterpret_cast<BufferEntry*>(out.Get());
- entry.denominator = static_cast<float>(denominator);
- entry.gamma = 0.0;
- for (unsigned i = 1; i <= 3; ++i) {
- entry.gamma += discount_.Get(i) * static_cast<float>(counts[i]);
- }
-
- // Makes model sum to 1 with pruning (I hope).
- entry.gamma += normalizer;
-
- entry.gamma /= entry.denominator;
-
- if(pruning_) {
- // If pruning is enabled the stream actually contains HashBufferEntry, see InitialProbabilities(...),
- // so add a hash value that identifies the current ngram.
- static_cast<HashBufferEntry*>(&entry)->hash_value = util::MurmurHashNative(previous_raw, size);
- }
- }
- out.Poison();
- }
-
- private:
- const Discount &discount_;
- const util::stream::ChainPosition input_;
- bool pruning_;
-};
-
-class MergeRight {
- public:
- MergeRight(bool interpolate_unigrams, const util::stream::ChainPosition &from_adder, const Discount &discount, const SpecialVocab &specials)
- : interpolate_unigrams_(interpolate_unigrams), from_adder_(from_adder), discount_(discount), specials_(specials) {}
-
- // calculate the initial probability of each n-gram (before order-interpolation)
- // Run() gets invoked once for each order
- void Run(const util::stream::ChainPosition &primary) {
- util::stream::Stream summed(from_adder_);
-
- PruneNGramStream grams(primary, specials_);
-
- // Without interpolation, the interpolation weight goes to <unk>.
- if (grams->Order() == 1) {
- BufferEntry sums(*static_cast<const BufferEntry*>(summed.Get()));
- // Special case for <unk>
- assert(*grams->begin() == kUNK);
- float gamma_assign;
- if (interpolate_unigrams_) {
- // Default: treat <unk> like a zeroton.
- gamma_assign = sums.gamma;
- grams->Value().uninterp.prob = 0.0;
- } else {
- // SRI: give all the interpolation mass to <unk>
- gamma_assign = 0.0;
- grams->Value().uninterp.prob = sums.gamma;
- }
- grams->Value().uninterp.gamma = gamma_assign;
-
- for (++grams; *grams->begin() != specials_.BOS(); ++grams) {
- grams->Value().uninterp.prob = discount_.Apply(grams->Value().count) / sums.denominator;
- grams->Value().uninterp.gamma = gamma_assign;
- }
-
- // Special case for <s>: probability 1.0. This allows <s> to be
- // explicitly scored as part of the sentence without impacting
- // probability and computes q correctly as b(<s>).
- assert(*grams->begin() == specials_.BOS());
- grams->Value().uninterp.prob = 1.0;
- grams->Value().uninterp.gamma = 0.0;
-
- while (++grams) {
- grams->Value().uninterp.prob = discount_.Apply(grams->Value().count) / sums.denominator;
- grams->Value().uninterp.gamma = gamma_assign;
- }
- ++summed;
- return;
- }
-
- std::vector<WordIndex> previous(grams->Order() - 1);
- const std::size_t size = sizeof(WordIndex) * previous.size();
- for (; grams; ++summed) {
- memcpy(&previous[0], grams->begin(), size);
- const BufferEntry &sums = *static_cast<const BufferEntry*>(summed.Get());
-
- do {
- BuildingPayload &pay = grams->Value();
- pay.uninterp.prob = discount_.Apply(grams->Value().UnmarkedCount()) / sums.denominator;
- pay.uninterp.gamma = sums.gamma;
- } while (++grams && !memcmp(&previous[0], grams->begin(), size));
- }
- }
-
- private:
- bool interpolate_unigrams_;
- util::stream::ChainPosition from_adder_;
- Discount discount_;
- const SpecialVocab specials_;
-};
-
-} // namespace
-
-void InitialProbabilities(
- const InitialProbabilitiesConfig &config,
- const std::vector<Discount> &discounts,
- util::stream::Chains &primary,
- util::stream::Chains &second_in,
- util::stream::Chains &gamma_out,
- const std::vector<uint64_t> &prune_thresholds,
- bool prune_vocab,
- const SpecialVocab &specials) {
- for (size_t i = 0; i < primary.size(); ++i) {
- util::stream::ChainConfig gamma_config = config.adder_out;
- if(prune_vocab || prune_thresholds[i] > 0)
- gamma_config.entry_size = sizeof(HashBufferEntry);
- else
- gamma_config.entry_size = sizeof(BufferEntry);
-
- util::stream::ChainPosition second(second_in[i].Add());
- second_in[i] >> util::stream::kRecycle;
- gamma_out.push_back(gamma_config);
- gamma_out[i] >> AddRight(discounts[i], second, prune_vocab || prune_thresholds[i] > 0);
-
- primary[i] >> MergeRight(config.interpolate_unigrams, gamma_out[i].Add(), discounts[i], specials);
-
- // Don't bother with the OnlyGamma thread for something to discard.
- if (i) gamma_out[i] >> OnlyGamma(prune_vocab || prune_thresholds[i] > 0);
- }
-}
-
-}} // namespaces
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/builder/initial_probabilities.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/builder/initial_probabilities.hh b/ext/kenlm/lm/builder/initial_probabilities.hh
deleted file mode 100644
index caeea58..0000000
--- a/ext/kenlm/lm/builder/initial_probabilities.hh
+++ /dev/null
@@ -1,45 +0,0 @@
-#ifndef LM_BUILDER_INITIAL_PROBABILITIES_H
-#define LM_BUILDER_INITIAL_PROBABILITIES_H
-
-#include "lm/builder/discount.hh"
-#include "lm/word_index.hh"
-#include "util/stream/config.hh"
-
-#include <vector>
-
-namespace util { namespace stream { class Chains; } }
-
-namespace lm {
-class SpecialVocab;
-namespace builder {
-
-struct InitialProbabilitiesConfig {
- // These should be small buffers to keep the adder from getting too far ahead
- util::stream::ChainConfig adder_in;
- util::stream::ChainConfig adder_out;
- // SRILM doesn't normally interpolate unigrams.
- bool interpolate_unigrams;
-};
-
-/* Compute initial (uninterpolated) probabilities
- * primary: the normal chain of n-grams. Incoming is context sorted adjusted
- * counts. Outgoing has uninterpolated probabilities for use by Interpolate.
- * second_in: a second copy of the primary input. Discard the output.
- * gamma_out: Computed gamma values are output on these chains in suffix order.
- * The values are bare floats and should be buffered for interpolation to
- * use.
- */
-void InitialProbabilities(
- const InitialProbabilitiesConfig &config,
- const std::vector<Discount> &discounts,
- util::stream::Chains &primary,
- util::stream::Chains &second_in,
- util::stream::Chains &gamma_out,
- const std::vector<uint64_t> &prune_thresholds,
- bool prune_vocab,
- const SpecialVocab &vocab);
-
-} // namespace builder
-} // namespace lm
-
-#endif // LM_BUILDER_INITIAL_PROBABILITIES_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/builder/interpolate.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/builder/interpolate.cc b/ext/kenlm/lm/builder/interpolate.cc
deleted file mode 100644
index a62ef43..0000000
--- a/ext/kenlm/lm/builder/interpolate.cc
+++ /dev/null
@@ -1,166 +0,0 @@
-#include "lm/builder/interpolate.hh"
-
-#include "lm/builder/hash_gamma.hh"
-#include "lm/builder/payload.hh"
-#include "lm/common/compare.hh"
-#include "lm/common/joint_order.hh"
-#include "lm/common/ngram_stream.hh"
-#include "lm/lm_exception.hh"
-#include "util/fixed_array.hh"
-#include "util/murmur_hash.hh"
-
-#include <iostream>
-#include <cassert>
-#include <cmath>
-
-namespace lm { namespace builder {
-namespace {
-
-/* Calculate q, the collapsed probability and backoff, as defined in
- * @inproceedings{Heafield-rest,
- * author = {Kenneth Heafield and Philipp Koehn and Alon Lavie},
- * title = {Language Model Rest Costs and Space-Efficient Storage},
- * year = {2012},
- * month = {July},
- * booktitle = {Proceedings of the Joint Conference on Empirical Methods in Natural Language Processing and Computational Natural Language Learning},
- * address = {Jeju Island, Korea},
- * pages = {1169--1178},
- * url = {http://kheafield.com/professional/edinburgh/rest\_paper.pdf},
- * }
- * This is particularly convenient to calculate during interpolation because
- * the needed backoff terms are already accessed at the same time.
- */
-class OutputQ {
- public:
- explicit OutputQ(std::size_t order) : q_delta_(order) {}
-
- void Gram(unsigned order_minus_1, float full_backoff, ProbBackoff &out) {
- float &q_del = q_delta_[order_minus_1];
- if (order_minus_1) {
- // Divide by context's backoff (which comes in as out.backoff)
- q_del = q_delta_[order_minus_1 - 1] / out.backoff * full_backoff;
- } else {
- q_del = full_backoff;
- }
- out.prob = log10f(out.prob * q_del);
- // TODO: stop wastefully outputting this!
- out.backoff = 0.0;
- }
-
- private:
- // Product of backoffs in the numerator divided by backoffs in the
- // denominator. Does not include
- std::vector<float> q_delta_;
-};
-
-/* Default: output probability and backoff */
-class OutputProbBackoff {
- public:
- explicit OutputProbBackoff(std::size_t /*order*/) {}
-
- void Gram(unsigned /*order_minus_1*/, float full_backoff, ProbBackoff &out) const {
- // Correcting for numerical precision issues. Take that IRST.
- out.prob = std::min(0.0f, log10f(out.prob));
- out.backoff = log10f(full_backoff);
- }
-};
-
-template <class Output> class Callback {
- public:
- Callback(float uniform_prob, const util::stream::ChainPositions &backoffs, const std::vector<uint64_t> &prune_thresholds, bool prune_vocab, const SpecialVocab &specials)
- : backoffs_(backoffs.size()), probs_(backoffs.size() + 2),
- prune_thresholds_(prune_thresholds),
- prune_vocab_(prune_vocab),
- output_(backoffs.size() + 1 /* order */),
- specials_(specials) {
- probs_[0] = uniform_prob;
- for (std::size_t i = 0; i < backoffs.size(); ++i) {
- backoffs_.push_back(backoffs[i]);
- }
- }
-
- ~Callback() {
- for (std::size_t i = 0; i < backoffs_.size(); ++i) {
- if(prune_vocab_ || prune_thresholds_[i + 1] > 0)
- while(backoffs_[i])
- ++backoffs_[i];
-
- if (backoffs_[i]) {
- std::cerr << "Backoffs do not match for order " << (i + 1) << std::endl;
- abort();
- }
- }
- }
-
- void Enter(unsigned order_minus_1, void *data) {
- NGram<BuildingPayload> gram(data, order_minus_1 + 1);
- BuildingPayload &pay = gram.Value();
- pay.complete.prob = pay.uninterp.prob + pay.uninterp.gamma * probs_[order_minus_1];
- probs_[order_minus_1 + 1] = pay.complete.prob;
-
- float out_backoff;
- if (order_minus_1 < backoffs_.size() && *(gram.end() - 1) != specials_.UNK() && *(gram.end() - 1) != specials_.EOS() && backoffs_[order_minus_1]) {
- if(prune_vocab_ || prune_thresholds_[order_minus_1 + 1] > 0) {
- //Compute hash value for current context
- uint64_t current_hash = util::MurmurHashNative(gram.begin(), gram.Order() * sizeof(WordIndex));
-
- const HashGamma *hashed_backoff = static_cast<const HashGamma*>(backoffs_[order_minus_1].Get());
- while(current_hash != hashed_backoff->hash_value && ++backoffs_[order_minus_1])
- hashed_backoff = static_cast<const HashGamma*>(backoffs_[order_minus_1].Get());
-
- if(current_hash == hashed_backoff->hash_value) {
- out_backoff = hashed_backoff->gamma;
- ++backoffs_[order_minus_1];
- } else {
- // Has been pruned away so it is not a context anymore
- out_backoff = 1.0;
- }
- } else {
- out_backoff = *static_cast<const float*>(backoffs_[order_minus_1].Get());
- ++backoffs_[order_minus_1];
- }
- } else {
- // Not a context.
- out_backoff = 1.0;
- }
-
- output_.Gram(order_minus_1, out_backoff, pay.complete);
- }
-
- void Exit(unsigned, void *) const {}
-
- private:
- util::FixedArray<util::stream::Stream> backoffs_;
-
- std::vector<float> probs_;
- const std::vector<uint64_t>& prune_thresholds_;
- bool prune_vocab_;
-
- Output output_;
- const SpecialVocab specials_;
-};
-} // namespace
-
-Interpolate::Interpolate(uint64_t vocab_size, const util::stream::ChainPositions &backoffs, const std::vector<uint64_t>& prune_thresholds, bool prune_vocab, bool output_q, const SpecialVocab &specials)
- : uniform_prob_(1.0 / static_cast<float>(vocab_size)), // Includes <unk> but excludes <s>.
- backoffs_(backoffs),
- prune_thresholds_(prune_thresholds),
- prune_vocab_(prune_vocab),
- output_q_(output_q),
- specials_(specials) {}
-
-// perform order-wise interpolation
-void Interpolate::Run(const util::stream::ChainPositions &positions) {
- assert(positions.size() == backoffs_.size() + 1);
- if (output_q_) {
- typedef Callback<OutputQ> C;
- C callback(uniform_prob_, backoffs_, prune_thresholds_, prune_vocab_, specials_);
- JointOrder<C, SuffixOrder>(positions, callback);
- } else {
- typedef Callback<OutputProbBackoff> C;
- C callback(uniform_prob_, backoffs_, prune_thresholds_, prune_vocab_, specials_);
- JointOrder<C, SuffixOrder>(positions, callback);
- }
-}
-
-}} // namespaces
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/builder/interpolate.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/builder/interpolate.hh b/ext/kenlm/lm/builder/interpolate.hh
deleted file mode 100644
index d20cd54..0000000
--- a/ext/kenlm/lm/builder/interpolate.hh
+++ /dev/null
@@ -1,37 +0,0 @@
-#ifndef LM_BUILDER_INTERPOLATE_H
-#define LM_BUILDER_INTERPOLATE_H
-
-#include "lm/common/special.hh"
-#include "lm/word_index.hh"
-#include "util/stream/multi_stream.hh"
-
-#include <vector>
-
-#include <stdint.h>
-
-namespace lm { namespace builder {
-
-/* Interpolate step.
- * Input: suffix sorted n-grams with (p_uninterpolated, gamma) from
- * InitialProbabilities.
- * Output: suffix sorted n-grams with complete probability
- */
-class Interpolate {
- public:
- // Normally vocab_size is the unigram count-1 (since p(<s>) = 0) but might
- // be larger when the user specifies a consistent vocabulary size.
- explicit Interpolate(uint64_t vocab_size, const util::stream::ChainPositions &backoffs, const std::vector<uint64_t> &prune_thresholds, bool prune_vocab, bool output_q, const SpecialVocab &specials);
-
- void Run(const util::stream::ChainPositions &positions);
-
- private:
- float uniform_prob_;
- util::stream::ChainPositions backoffs_;
- const std::vector<uint64_t> prune_thresholds_;
- bool prune_vocab_;
- bool output_q_;
- const SpecialVocab specials_;
-};
-
-}} // namespaces
-#endif // LM_BUILDER_INTERPOLATE_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/builder/lmplz_main.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/builder/lmplz_main.cc b/ext/kenlm/lm/builder/lmplz_main.cc
deleted file mode 100644
index cc3f381..0000000
--- a/ext/kenlm/lm/builder/lmplz_main.cc
+++ /dev/null
@@ -1,220 +0,0 @@
-#include "lm/builder/output.hh"
-#include "lm/builder/pipeline.hh"
-#include "lm/common/size_option.hh"
-#include "lm/lm_exception.hh"
-#include "util/file.hh"
-#include "util/file_piece.hh"
-#include "util/usage.hh"
-
-#include <iostream>
-
-#include <boost/program_options.hpp>
-#include <boost/version.hpp>
-#include <vector>
-
-namespace {
-
-// Parse and validate pruning thresholds then return vector of threshold counts
-// for each n-grams order.
-std::vector<uint64_t> ParsePruning(const std::vector<std::string> ¶m, std::size_t order) {
- // convert to vector of integers
- std::vector<uint64_t> prune_thresholds;
- prune_thresholds.reserve(order);
- for (std::vector<std::string>::const_iterator it(param.begin()); it != param.end(); ++it) {
- try {
- prune_thresholds.push_back(boost::lexical_cast<uint64_t>(*it));
- } catch(const boost::bad_lexical_cast &) {
- UTIL_THROW(util::Exception, "Bad pruning threshold " << *it);
- }
- }
-
- // Fill with zeros by default.
- if (prune_thresholds.empty()) {
- prune_thresholds.resize(order, 0);
- return prune_thresholds;
- }
-
- // validate pruning threshold if specified
- // throw if each n-gram order has not threshold specified
- UTIL_THROW_IF(prune_thresholds.size() > order, util::Exception, "You specified pruning thresholds for orders 1 through " << prune_thresholds.size() << " but the model only has order " << order);
- // threshold for unigram can only be 0 (no pruning)
-
- // check if threshold are not in decreasing order
- uint64_t lower_threshold = 0;
- for (std::vector<uint64_t>::iterator it = prune_thresholds.begin(); it != prune_thresholds.end(); ++it) {
- UTIL_THROW_IF(lower_threshold > *it, util::Exception, "Pruning thresholds should be in non-decreasing order. Otherwise substrings would be removed, which is bad for query-time data structures.");
- lower_threshold = *it;
- }
-
- // Pad to all orders using the last value.
- prune_thresholds.resize(order, prune_thresholds.back());
- return prune_thresholds;
-}
-
-lm::builder::Discount ParseDiscountFallback(const std::vector<std::string> ¶m) {
- lm::builder::Discount ret;
- UTIL_THROW_IF(param.size() > 3, util::Exception, "Specify at most three fallback discounts: 1, 2, and 3+");
- UTIL_THROW_IF(param.empty(), util::Exception, "Fallback discounting enabled, but no discount specified");
- ret.amount[0] = 0.0;
- for (unsigned i = 0; i < 3; ++i) {
- float discount = boost::lexical_cast<float>(param[i < param.size() ? i : (param.size() - 1)]);
- UTIL_THROW_IF(discount < 0.0 || discount > static_cast<float>(i+1), util::Exception, "The discount for count " << (i+1) << " was parsed as " << discount << " which is not in the range [0, " << (i+1) << "].");
- ret.amount[i + 1] = discount;
- }
- return ret;
-}
-
-} // namespace
-
-int main(int argc, char *argv[]) {
- try {
- namespace po = boost::program_options;
- po::options_description options("Language model building options");
- lm::builder::PipelineConfig pipeline;
-
- std::string text, intermediate, arpa;
- std::vector<std::string> pruning;
- std::vector<std::string> discount_fallback;
- std::vector<std::string> discount_fallback_default;
- discount_fallback_default.push_back("0.5");
- discount_fallback_default.push_back("1");
- discount_fallback_default.push_back("1.5");
- bool verbose_header;
-
- options.add_options()
- ("help,h", po::bool_switch(), "Show this help message")
- ("order,o", po::value<std::size_t>(&pipeline.order)
-#if BOOST_VERSION >= 104200
- ->required()
-#endif
- , "Order of the model")
- ("interpolate_unigrams", po::value<bool>(&pipeline.initial_probs.interpolate_unigrams)->default_value(true)->implicit_value(true), "Interpolate the unigrams (default) as opposed to giving lots of mass to <unk> like SRI. If you want SRI's behavior with a large <unk> and the old lmplz default, use --interpolate_unigrams 0.")
- ("skip_symbols", po::bool_switch(), "Treat <s>, </s>, and <unk> as whitespace instead of throwing an exception")
- ("temp_prefix,T", po::value<std::string>(&pipeline.sort.temp_prefix)->default_value("/tmp/lm"), "Temporary file prefix")
- ("memory,S", lm:: SizeOption(pipeline.sort.total_memory, util::GuessPhysicalMemory() ? "80%" : "1G"), "Sorting memory")
- ("minimum_block", lm::SizeOption(pipeline.minimum_block, "8K"), "Minimum block size to allow")
- ("sort_block", lm::SizeOption(pipeline.sort.buffer_size, "64M"), "Size of IO operations for sort (determines arity)")
- ("block_count", po::value<std::size_t>(&pipeline.block_count)->default_value(2), "Block count (per order)")
- ("vocab_estimate", po::value<lm::WordIndex>(&pipeline.vocab_estimate)->default_value(1000000), "Assume this vocabulary size for purposes of calculating memory in step 1 (corpus count) and pre-sizing the hash table")
- ("vocab_pad", po::value<uint64_t>(&pipeline.vocab_size_for_unk)->default_value(0), "If the vocabulary is smaller than this value, pad with <unk> to reach this size. Requires --interpolate_unigrams")
- ("verbose_header", po::bool_switch(&verbose_header), "Add a verbose header to the ARPA file that includes information such as token count, smoothing type, etc.")
- ("text", po::value<std::string>(&text), "Read text from a file instead of stdin")
- ("arpa", po::value<std::string>(&arpa), "Write ARPA to a file instead of stdout")
- ("intermediate", po::value<std::string>(&intermediate), "Write ngrams to intermediate files. Turns off ARPA output (which can be reactivated by --arpa file). Forces --renumber on.")
- ("renumber", po::bool_switch(&pipeline.renumber_vocabulary), "Rrenumber the vocabulary identifiers so that they are monotone with the hash of each string. This is consistent with the ordering used by the trie data structure.")
- ("collapse_values", po::bool_switch(&pipeline.output_q), "Collapse probability and backoff into a single value, q that yields the same sentence-level probabilities. See http://kheafield.com/professional/edinburgh/rest_paper.pdf for more details, including a proof.")
- ("prune", po::value<std::vector<std::string> >(&pruning)->multitoken(), "Prune n-grams with count less than or equal to the given threshold. Specify one value for each order i.e. 0 0 1 to prune singleton trigrams and above. The sequence of values must be non-decreasing and the last value applies to any remaining orders. Default is to not prune, which is equivalent to --prune 0.")
- ("limit_vocab_file", po::value<std::string>(&pipeline.prune_vocab_file)->default_value(""), "Read allowed vocabulary separated by whitespace. N-grams that contain vocabulary items not in this list will be pruned. Can be combined with --prune arg")
- ("discount_fallback", po::value<std::vector<std::string> >(&discount_fallback)->multitoken()->implicit_value(discount_fallback_default, "0.5 1 1.5"), "The closed-form estimate for Kneser-Ney discounts does not work without singletons or doubletons. It can also fail if these values are out of range. This option falls back to user-specified discounts when the closed-form estimate fails. Note that this option is generally a bad idea: you should deduplicate your corpus instead. However, class-based models need custom discounts because they lack singleton unigrams. Provide up to three discounts (for adjusted counts 1, 2, and 3+), which will be applied to all orders where the closed-form estimates fail.");
- po::variables_map vm;
- po::store(po::parse_command_line(argc, argv, options), vm);
-
- if (argc == 1 || vm["help"].as<bool>()) {
- std::cerr <<
- "Builds unpruned language models with modified Kneser-Ney smoothing.\n\n"
- "Please cite:\n"
- "@inproceedings{Heafield-estimate,\n"
- " author = {Kenneth Heafield and Ivan Pouzyrevsky and Jonathan H. Clark and Philipp Koehn},\n"
- " title = {Scalable Modified {Kneser-Ney} Language Model Estimation},\n"
- " year = {2013},\n"
- " month = {8},\n"
- " booktitle = {Proceedings of the 51st Annual Meeting of the Association for Computational Linguistics},\n"
- " address = {Sofia, Bulgaria},\n"
- " url = {http://kheafield.com/professional/edinburgh/estimate\\_paper.pdf},\n"
- "}\n\n"
- "Provide the corpus on stdin. The ARPA file will be written to stdout. Order of\n"
- "the model (-o) is the only mandatory option. As this is an on-disk program,\n"
- "setting the temporary file location (-T) and sorting memory (-S) is recommended.\n\n"
- "Memory sizes are specified like GNU sort: a number followed by a unit character.\n"
- "Valid units are \% for percentage of memory (supported platforms only) and (in\n"
- "increasing powers of 1024): b, K, M, G, T, P, E, Z, Y. Default is K (*1024).\n";
- uint64_t mem = util::GuessPhysicalMemory();
- if (mem) {
- std::cerr << "This machine has " << mem << " bytes of memory.\n\n";
- } else {
- std::cerr << "Unable to determine the amount of memory on this machine.\n\n";
- }
- std::cerr << options << std::endl;
- return 1;
- }
-
- po::notify(vm);
-
- // required() appeared in Boost 1.42.0.
-#if BOOST_VERSION < 104200
- if (!vm.count("order")) {
- std::cerr << "the option '--order' is required but missing" << std::endl;
- return 1;
- }
-#endif
-
- if (pipeline.vocab_size_for_unk && !pipeline.initial_probs.interpolate_unigrams) {
- std::cerr << "--vocab_pad requires --interpolate_unigrams be on" << std::endl;
- return 1;
- }
-
- if (vm["skip_symbols"].as<bool>()) {
- pipeline.disallowed_symbol_action = lm::COMPLAIN;
- } else {
- pipeline.disallowed_symbol_action = lm::THROW_UP;
- }
-
- if (vm.count("discount_fallback")) {
- pipeline.discount.fallback = ParseDiscountFallback(discount_fallback);
- pipeline.discount.bad_action = lm::COMPLAIN;
- } else {
- // Unused, just here to prevent the compiler from complaining about uninitialized.
- pipeline.discount.fallback = lm::builder::Discount();
- pipeline.discount.bad_action = lm::THROW_UP;
- }
-
- // parse pruning thresholds. These depend on order, so it is not done as a notifier.
- pipeline.prune_thresholds = ParsePruning(pruning, pipeline.order);
-
- if (!vm["limit_vocab_file"].as<std::string>().empty()) {
- pipeline.prune_vocab = true;
- }
- else {
- pipeline.prune_vocab = false;
- }
-
- util::NormalizeTempPrefix(pipeline.sort.temp_prefix);
-
- lm::builder::InitialProbabilitiesConfig &initial = pipeline.initial_probs;
- // TODO: evaluate options for these.
- initial.adder_in.total_memory = 32768;
- initial.adder_in.block_count = 2;
- initial.adder_out.total_memory = 32768;
- initial.adder_out.block_count = 2;
- pipeline.read_backoffs = initial.adder_out;
-
- // Read from stdin, write to stdout by default
- util::scoped_fd in(0), out(1);
- if (vm.count("text")) {
- in.reset(util::OpenReadOrThrow(text.c_str()));
- }
- if (vm.count("arpa")) {
- out.reset(util::CreateOrThrow(arpa.c_str()));
- }
-
- try {
- bool writing_intermediate = vm.count("intermediate");
- if (writing_intermediate) {
- pipeline.renumber_vocabulary = true;
- }
- lm::builder::Output output(writing_intermediate ? intermediate : pipeline.sort.temp_prefix, writing_intermediate, pipeline.output_q);
- if (!writing_intermediate || vm.count("arpa")) {
- output.Add(new lm::builder::PrintHook(out.release(), verbose_header));
- }
- lm::builder::Pipeline(pipeline, in.release(), output);
- } catch (const util::MallocException &e) {
- std::cerr << e.what() << std::endl;
- std::cerr << "Try rerunning with a more conservative -S setting than " << vm["memory"].as<std::string>() << std::endl;
- return 1;
- }
- util::PrintUsage(std::cerr);
- } catch (const std::exception &e) {
- std::cerr << e.what() << std::endl;
- return 1;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/builder/output.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/builder/output.cc b/ext/kenlm/lm/builder/output.cc
deleted file mode 100644
index 604fa22..0000000
--- a/ext/kenlm/lm/builder/output.cc
+++ /dev/null
@@ -1,52 +0,0 @@
-#include "lm/builder/output.hh"
-
-#include "lm/common/model_buffer.hh"
-#include "lm/common/print.hh"
-#include "util/file_stream.hh"
-#include "util/stream/multi_stream.hh"
-
-#include <iostream>
-
-namespace lm { namespace builder {
-
-OutputHook::~OutputHook() {}
-
-Output::Output(StringPiece file_base, bool keep_buffer, bool output_q)
- : buffer_(file_base, keep_buffer, output_q) {}
-
-void Output::SinkProbs(util::stream::Chains &chains) {
- Apply(PROB_PARALLEL_HOOK, chains);
- if (!buffer_.Keep() && !Have(PROB_SEQUENTIAL_HOOK)) {
- chains >> util::stream::kRecycle;
- chains.Wait(true);
- return;
- }
- buffer_.Sink(chains, header_.counts_pruned);
- chains >> util::stream::kRecycle;
- chains.Wait(false);
- if (Have(PROB_SEQUENTIAL_HOOK)) {
- std::cerr << "=== 5/5 Writing ARPA model ===" << std::endl;
- buffer_.Source(chains);
- Apply(PROB_SEQUENTIAL_HOOK, chains);
- chains >> util::stream::kRecycle;
- chains.Wait(true);
- }
-}
-
-void Output::Apply(HookType hook_type, util::stream::Chains &chains) {
- for (boost::ptr_vector<OutputHook>::iterator entry = outputs_[hook_type].begin(); entry != outputs_[hook_type].end(); ++entry) {
- entry->Sink(header_, VocabFile(), chains);
- }
-}
-
-void PrintHook::Sink(const HeaderInfo &info, int vocab_file, util::stream::Chains &chains) {
- if (verbose_header_) {
- util::FileStream out(file_.get(), 50);
- out << "# Input file: " << info.input_file << '\n';
- out << "# Token count: " << info.token_count << '\n';
- out << "# Smoothing: Modified Kneser-Ney" << '\n';
- }
- chains >> PrintARPA(vocab_file, file_.get(), info.counts_pruned);
-}
-
-}} // namespaces
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/builder/output.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/builder/output.hh b/ext/kenlm/lm/builder/output.hh
deleted file mode 100644
index 69d6c6d..0000000
--- a/ext/kenlm/lm/builder/output.hh
+++ /dev/null
@@ -1,85 +0,0 @@
-#ifndef LM_BUILDER_OUTPUT_H
-#define LM_BUILDER_OUTPUT_H
-
-#include "lm/builder/header_info.hh"
-#include "lm/common/model_buffer.hh"
-#include "util/file.hh"
-
-#include <boost/ptr_container/ptr_vector.hpp>
-#include <boost/utility.hpp>
-
-namespace util { namespace stream { class Chains; class ChainPositions; } }
-
-/* Outputs from lmplz: ARPA, sharded files, etc */
-namespace lm { namespace builder {
-
-// These are different types of hooks. Values should be consecutive to enable a vector lookup.
-enum HookType {
- // TODO: counts.
- PROB_PARALLEL_HOOK, // Probability and backoff (or just q). Output must process the orders in parallel or there will be a deadlock.
- PROB_SEQUENTIAL_HOOK, // Probability and backoff (or just q). Output can process orders any way it likes. This requires writing the data to disk then reading. Useful for ARPA files, which put unigrams first etc.
- NUMBER_OF_HOOKS // Keep this last so we know how many values there are.
-};
-
-class OutputHook {
- public:
- explicit OutputHook(HookType hook_type) : type_(hook_type) {}
-
- virtual ~OutputHook();
-
- virtual void Sink(const HeaderInfo &info, int vocab_file, util::stream::Chains &chains) = 0;
-
- HookType Type() const { return type_; }
-
- private:
- HookType type_;
-};
-
-class Output : boost::noncopyable {
- public:
- Output(StringPiece file_base, bool keep_buffer, bool output_q);
-
- // Takes ownership.
- void Add(OutputHook *hook) {
- outputs_[hook->Type()].push_back(hook);
- }
-
- bool Have(HookType hook_type) const {
- return !outputs_[hook_type].empty();
- }
-
- int VocabFile() const { return buffer_.VocabFile(); }
-
- void SetHeader(const HeaderInfo &header) { header_ = header; }
- const HeaderInfo &GetHeader() const { return header_; }
-
- // This is called by the pipeline.
- void SinkProbs(util::stream::Chains &chains);
-
- unsigned int Steps() const { return Have(PROB_SEQUENTIAL_HOOK); }
-
- private:
- void Apply(HookType hook_type, util::stream::Chains &chains);
-
- ModelBuffer buffer_;
-
- boost::ptr_vector<OutputHook> outputs_[NUMBER_OF_HOOKS];
- HeaderInfo header_;
-};
-
-class PrintHook : public OutputHook {
- public:
- // Takes ownership
- PrintHook(int write_fd, bool verbose_header)
- : OutputHook(PROB_SEQUENTIAL_HOOK), file_(write_fd), verbose_header_(verbose_header) {}
-
- void Sink(const HeaderInfo &info, int vocab_file, util::stream::Chains &chains);
-
- private:
- util::scoped_fd file_;
- bool verbose_header_;
-};
-
-}} // namespaces
-
-#endif // LM_BUILDER_OUTPUT_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/builder/payload.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/builder/payload.hh b/ext/kenlm/lm/builder/payload.hh
deleted file mode 100644
index ba12725..0000000
--- a/ext/kenlm/lm/builder/payload.hh
+++ /dev/null
@@ -1,48 +0,0 @@
-#ifndef LM_BUILDER_PAYLOAD_H
-#define LM_BUILDER_PAYLOAD_H
-
-#include "lm/weights.hh"
-#include "lm/word_index.hh"
-#include <stdint.h>
-
-namespace lm { namespace builder {
-
-struct Uninterpolated {
- float prob; // Uninterpolated probability.
- float gamma; // Interpolation weight for lower order.
-};
-
-union BuildingPayload {
- uint64_t count;
- Uninterpolated uninterp;
- ProbBackoff complete;
-
- /*mjd**********************************************************************/
- bool IsMarked() const {
- return count >> (sizeof(count) * 8 - 1);
- }
-
- void Mark() {
- count |= (1ul << (sizeof(count) * 8 - 1));
- }
-
- void Unmark() {
- count &= ~(1ul << (sizeof(count) * 8 - 1));
- }
-
- uint64_t UnmarkedCount() const {
- return count & ~(1ul << (sizeof(count) * 8 - 1));
- }
-
- uint64_t CutoffCount() const {
- return IsMarked() ? 0 : UnmarkedCount();
- }
- /*mjd**********************************************************************/
-};
-
-const WordIndex kBOS = 1;
-const WordIndex kEOS = 2;
-
-}} // namespaces
-
-#endif // LM_BUILDER_PAYLOAD_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/builder/pipeline.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/builder/pipeline.cc b/ext/kenlm/lm/builder/pipeline.cc
deleted file mode 100644
index 64e30f7..0000000
--- a/ext/kenlm/lm/builder/pipeline.cc
+++ /dev/null
@@ -1,385 +0,0 @@
-#include "lm/builder/pipeline.hh"
-
-#include "lm/builder/adjust_counts.hh"
-#include "lm/builder/combine_counts.hh"
-#include "lm/builder/corpus_count.hh"
-#include "lm/builder/hash_gamma.hh"
-#include "lm/builder/initial_probabilities.hh"
-#include "lm/builder/interpolate.hh"
-#include "lm/builder/output.hh"
-#include "lm/common/compare.hh"
-#include "lm/common/renumber.hh"
-
-#include "lm/sizes.hh"
-#include "lm/vocab.hh"
-
-#include "util/exception.hh"
-#include "util/file.hh"
-#include "util/stream/io.hh"
-
-#include <algorithm>
-#include <iostream>
-#include <fstream>
-#include <vector>
-
-namespace lm { namespace builder {
-
-using util::stream::Sorts;
-
-namespace {
-
-void PrintStatistics(const std::vector<uint64_t> &counts, const std::vector<uint64_t> &counts_pruned, const std::vector<Discount> &discounts) {
- std::cerr << "Statistics:\n";
- for (size_t i = 0; i < counts.size(); ++i) {
- std::cerr << (i + 1) << ' ' << counts_pruned[i];
- if(counts[i] != counts_pruned[i])
- std::cerr << "/" << counts[i];
-
- for (size_t d = 1; d <= 3; ++d)
- std::cerr << " D" << d << (d == 3 ? "+=" : "=") << discounts[i].amount[d];
- std::cerr << '\n';
- }
-}
-
-class Master {
- public:
- explicit Master(PipelineConfig &config, unsigned output_steps)
- : config_(config), chains_(config.order), unigrams_(util::MakeTemp(config_.TempPrefix())), steps_(output_steps + 4) {
- config_.minimum_block = std::max(NGram<BuildingPayload>::TotalSize(config_.order), config_.minimum_block);
- }
-
- const PipelineConfig &Config() const { return config_; }
-
- util::stream::Chains &MutableChains() { return chains_; }
-
- template <class T> Master &operator>>(const T &worker) {
- chains_ >> worker;
- return *this;
- }
-
- // This takes the (partially) sorted ngrams and sets up for adjusted counts.
- void InitForAdjust(util::stream::Sort<SuffixOrder, CombineCounts> &ngrams, WordIndex types, std::size_t subtract_for_numbering) {
- const std::size_t each_order_min = config_.minimum_block * config_.block_count;
- // We know how many unigrams there are. Don't allocate more than needed to them.
- const std::size_t min_chains = (config_.order - 1) * each_order_min +
- std::min(types * NGram<BuildingPayload>::TotalSize(1), each_order_min);
- // Prevent overflow in subtracting.
- const std::size_t total = std::max<std::size_t>(config_.TotalMemory(), min_chains + subtract_for_numbering + config_.minimum_block);
- // Do merge sort with calculated laziness.
- const std::size_t merge_using = ngrams.Merge(std::min(total - min_chains - subtract_for_numbering, ngrams.DefaultLazy()));
-
- std::vector<uint64_t> count_bounds(1, types);
- CreateChains(total - merge_using - subtract_for_numbering, count_bounds);
- ngrams.Output(chains_.back(), merge_using);
- }
-
- // For initial probabilities, but this is generic.
- void SortAndReadTwice(const std::vector<uint64_t> &counts, Sorts<ContextOrder> &sorts, util::stream::Chains &second, util::stream::ChainConfig second_config) {
- bool unigrams_are_sorted = !config_.renumber_vocabulary;
- // Do merge first before allocating chain memory.
- for (std::size_t i = 0; i < config_.order - unigrams_are_sorted; ++i) {
- sorts[i].Merge(0);
- }
- // There's no lazy merge, so just divide memory amongst the chains.
- CreateChains(config_.TotalMemory(), counts);
- chains_.back().ActivateProgress();
- if (unigrams_are_sorted) {
- chains_[0] >> unigrams_.Source();
- second_config.entry_size = NGram<BuildingPayload>::TotalSize(1);
- second.push_back(second_config);
- second.back() >> unigrams_.Source();
- }
- for (std::size_t i = unigrams_are_sorted; i < config_.order; ++i) {
- util::scoped_fd fd(sorts[i - unigrams_are_sorted].StealCompleted());
- chains_[i].SetProgressTarget(util::SizeOrThrow(fd.get()));
- chains_[i] >> util::stream::PRead(util::DupOrThrow(fd.get()), true);
- second_config.entry_size = NGram<BuildingPayload>::TotalSize(i + 1);
- second.push_back(second_config);
- second.back() >> util::stream::PRead(fd.release(), true);
- }
- }
-
- // There is no sort after this, so go for broke on lazy merging.
- template <class Compare> void MaximumLazyInput(const std::vector<uint64_t> &counts, Sorts<Compare> &sorts) {
- // Determine the minimum we can use for all the chains.
- std::size_t min_chains = 0;
- for (std::size_t i = 0; i < config_.order; ++i) {
- min_chains += std::min(counts[i] * NGram<BuildingPayload>::TotalSize(i + 1), static_cast<uint64_t>(config_.minimum_block));
- }
- std::size_t for_merge = min_chains > config_.TotalMemory() ? 0 : (config_.TotalMemory() - min_chains);
- std::vector<std::size_t> laziness;
- // Prioritize longer n-grams.
- for (util::stream::Sort<SuffixOrder> *i = sorts.end() - 1; i >= sorts.begin(); --i) {
- laziness.push_back(i->Merge(for_merge));
- assert(for_merge >= laziness.back());
- for_merge -= laziness.back();
- }
- std::reverse(laziness.begin(), laziness.end());
-
- CreateChains(for_merge + min_chains, counts);
- chains_.back().ActivateProgress();
- chains_[0] >> unigrams_.Source();
- for (std::size_t i = 1; i < config_.order; ++i) {
- sorts[i - 1].Output(chains_[i], laziness[i - 1]);
- }
- }
-
- template <class Compare> void SetupSorts(Sorts<Compare> &sorts, bool exclude_unigrams) {
- sorts.Init(config_.order - exclude_unigrams);
- // Unigrams don't get sorted because their order is always the same.
- if (exclude_unigrams) chains_[0] >> unigrams_.Sink();
- for (std::size_t i = exclude_unigrams; i < config_.order; ++i) {
- sorts.push_back(chains_[i], config_.sort, Compare(i + 1));
- }
- chains_.Wait(true);
- }
-
- unsigned int Steps() const { return steps_; }
-
- private:
- // Create chains, allocating memory to them. Totally heuristic. Count
- // bounds are upper bounds on the counts or not present.
- void CreateChains(std::size_t remaining_mem, const std::vector<uint64_t> &count_bounds) {
- std::vector<std::size_t> assignments;
- assignments.reserve(config_.order);
- // Start by assigning maximum memory usage (to be refined later).
- for (std::size_t i = 0; i < count_bounds.size(); ++i) {
- assignments.push_back(static_cast<std::size_t>(std::min(
- static_cast<uint64_t>(remaining_mem),
- count_bounds[i] * static_cast<uint64_t>(NGram<BuildingPayload>::TotalSize(i + 1)))));
- }
- assignments.resize(config_.order, remaining_mem);
-
- // Now we know how much memory everybody wants. How much will they get?
- // Proportional to this.
- std::vector<float> portions;
- // Indices of orders that have yet to be assigned.
- std::vector<std::size_t> unassigned;
- for (std::size_t i = 0; i < config_.order; ++i) {
- portions.push_back(static_cast<float>((i+1) * NGram<BuildingPayload>::TotalSize(i+1)));
- unassigned.push_back(i);
- }
- /*If somebody doesn't eat their full dinner, give it to the rest of the
- * family. Then somebody else might not eat their full dinner etc. Ends
- * when everybody unassigned is hungry.
- */
- float sum;
- bool found_more;
- std::vector<std::size_t> block_count(config_.order);
- do {
- sum = 0.0;
- for (std::size_t i = 0; i < unassigned.size(); ++i) {
- sum += portions[unassigned[i]];
- }
- found_more = false;
- // If the proportional assignment is more than needed, give it just what it needs.
- for (std::vector<std::size_t>::iterator i = unassigned.begin(); i != unassigned.end();) {
- if (assignments[*i] <= remaining_mem * (portions[*i] / sum)) {
- remaining_mem -= assignments[*i];
- block_count[*i] = 1;
- i = unassigned.erase(i);
- found_more = true;
- } else {
- ++i;
- }
- }
- } while (found_more);
- for (std::vector<std::size_t>::iterator i = unassigned.begin(); i != unassigned.end(); ++i) {
- assignments[*i] = remaining_mem * (portions[*i] / sum);
- block_count[*i] = config_.block_count;
- }
- chains_.clear();
- std::cerr << "Chain sizes:";
- for (std::size_t i = 0; i < config_.order; ++i) {
- // Always have enough for at least one record.
- // This was crashing if e.g. there was no 5-gram.
- assignments[i] = std::max(assignments[i], block_count[i] * NGram<BuildingPayload>::TotalSize(i + 1));
- std::cerr << ' ' << (i+1) << ":" << assignments[i];
- chains_.push_back(util::stream::ChainConfig(NGram<BuildingPayload>::TotalSize(i + 1), block_count[i], assignments[i]));
- }
- std::cerr << std::endl;
- }
-
- PipelineConfig &config_;
-
- util::stream::Chains chains_;
-
- util::stream::FileBuffer unigrams_;
-
- const unsigned int steps_;
-};
-
-util::stream::Sort<SuffixOrder, CombineCounts> *CountText(int text_file /* input */, int vocab_file /* output */, Master &master, uint64_t &token_count, WordIndex &type_count, std::string &text_file_name, std::vector<bool> &prune_words) {
- const PipelineConfig &config = master.Config();
- std::cerr << "=== 1/" << master.Steps() << " Counting and sorting n-grams ===" << std::endl;
-
- const std::size_t vocab_usage = CorpusCount::VocabUsage(config.vocab_estimate);
- UTIL_THROW_IF(config.TotalMemory() < vocab_usage, util::Exception, "Vocab hash size estimate " << vocab_usage << " exceeds total memory " << config.TotalMemory());
- std::size_t memory_for_chain =
- // This much memory to work with after vocab hash table.
- static_cast<float>(config.TotalMemory() - vocab_usage) /
- // Solve for block size including the dedupe multiplier for one block.
- (static_cast<float>(config.block_count) + CorpusCount::DedupeMultiplier(config.order)) *
- // Chain likes memory expressed in terms of total memory.
- static_cast<float>(config.block_count);
- util::stream::Chain chain(util::stream::ChainConfig(NGram<BuildingPayload>::TotalSize(config.order), config.block_count, memory_for_chain));
-
- type_count = config.vocab_estimate;
- util::FilePiece text(text_file, NULL, &std::cerr);
- text_file_name = text.FileName();
- CorpusCount counter(text, vocab_file, token_count, type_count, prune_words, config.prune_vocab_file, chain.BlockSize() / chain.EntrySize(), config.disallowed_symbol_action);
- chain >> boost::ref(counter);
-
- util::scoped_ptr<util::stream::Sort<SuffixOrder, CombineCounts> > sorter(new util::stream::Sort<SuffixOrder, CombineCounts>(chain, config.sort, SuffixOrder(config.order), CombineCounts()));
- chain.Wait(true);
- return sorter.release();
-}
-
-void InitialProbabilities(const std::vector<uint64_t> &counts, const std::vector<uint64_t> &counts_pruned, const std::vector<Discount> &discounts, Master &master, Sorts<SuffixOrder> &primary, util::FixedArray<util::stream::FileBuffer> &gammas, const std::vector<uint64_t> &prune_thresholds, bool prune_vocab, const SpecialVocab &specials) {
- const PipelineConfig &config = master.Config();
- util::stream::Chains second(config.order);
-
- {
- Sorts<ContextOrder> sorts;
- master.SetupSorts(sorts, !config.renumber_vocabulary);
- PrintStatistics(counts, counts_pruned, discounts);
- lm::ngram::ShowSizes(counts_pruned);
- std::cerr << "=== 3/" << master.Steps() << " Calculating and sorting initial probabilities ===" << std::endl;
- master.SortAndReadTwice(counts_pruned, sorts, second, config.initial_probs.adder_in);
- }
-
- util::stream::Chains gamma_chains(config.order);
- InitialProbabilities(config.initial_probs, discounts, master.MutableChains(), second, gamma_chains, prune_thresholds, prune_vocab, specials);
- // Don't care about gamma for 0.
- gamma_chains[0] >> util::stream::kRecycle;
- gammas.Init(config.order - 1);
- for (std::size_t i = 1; i < config.order; ++i) {
- gammas.push_back(util::MakeTemp(config.TempPrefix()));
- gamma_chains[i] >> gammas[i - 1].Sink();
- }
- // Has to be done here due to gamma_chains scope.
- master.SetupSorts(primary, true);
-}
-
-void InterpolateProbabilities(const std::vector<uint64_t> &counts, Master &master, Sorts<SuffixOrder> &primary, util::FixedArray<util::stream::FileBuffer> &gammas, Output &output, const SpecialVocab &specials) {
- std::cerr << "=== 4/" << master.Steps() << " Calculating and writing order-interpolated probabilities ===" << std::endl;
- const PipelineConfig &config = master.Config();
- master.MaximumLazyInput(counts, primary);
-
- util::stream::Chains gamma_chains(config.order - 1);
- for (std::size_t i = 0; i < config.order - 1; ++i) {
- util::stream::ChainConfig read_backoffs(config.read_backoffs);
-
- if(config.prune_vocab || config.prune_thresholds[i + 1] > 0)
- read_backoffs.entry_size = sizeof(HashGamma);
- else
- read_backoffs.entry_size = sizeof(float);
-
- gamma_chains.push_back(read_backoffs);
- gamma_chains.back() >> gammas[i].Source(true);
- }
- master >> Interpolate(std::max(master.Config().vocab_size_for_unk, counts[0] - 1 /* <s> is not included */), util::stream::ChainPositions(gamma_chains), config.prune_thresholds, config.prune_vocab, config.output_q, specials);
- gamma_chains >> util::stream::kRecycle;
- output.SinkProbs(master.MutableChains());
-}
-
-class VocabNumbering {
- public:
- VocabNumbering(int final_vocab, StringPiece temp_prefix, bool renumber)
- : final_vocab_(final_vocab),
- renumber_(renumber),
- specials_(kBOS, kEOS) {
- if (renumber) {
- temporary_.reset(util::MakeTemp(temp_prefix));
- }
- }
-
- int WriteOnTheFly() const { return renumber_ ? temporary_.get() : final_vocab_; }
-
- // Compute the vocabulary mapping and return the memory used.
- std::size_t ComputeMapping(WordIndex type_count) {
- if (!renumber_) return 0;
- ngram::SortedVocabulary::ComputeRenumbering(type_count, temporary_.get(), final_vocab_, vocab_mapping_);
- temporary_.reset();
- return sizeof(WordIndex) * vocab_mapping_.size();
- }
-
- void ApplyRenumber(util::stream::Chains &chains) {
- if (!renumber_) return;
- for (std::size_t i = 0; i < chains.size(); ++i) {
- chains[i] >> Renumber(&*vocab_mapping_.begin(), i + 1);
- }
- specials_ = SpecialVocab(vocab_mapping_[specials_.BOS()], vocab_mapping_[specials_.EOS()]);
- }
-
- const SpecialVocab &Specials() const { return specials_; }
-
- private:
- int final_vocab_;
- // Out of order vocab file created on the fly.
- util::scoped_fd temporary_;
-
- bool renumber_;
-
- std::vector<WordIndex> vocab_mapping_;
-
- SpecialVocab specials_;
-};
-
-} // namespace
-
-void Pipeline(PipelineConfig &config, int text_file, Output &output) {
- // Some fail-fast sanity checks.
- if (config.sort.buffer_size * 4 > config.TotalMemory()) {
- config.sort.buffer_size = config.TotalMemory() / 4;
- std::cerr << "Warning: changing sort block size to " << config.sort.buffer_size << " bytes due to low total memory." << std::endl;
- }
- if (config.minimum_block < NGram<BuildingPayload>::TotalSize(config.order)) {
- config.minimum_block = NGram<BuildingPayload>::TotalSize(config.order);
- std::cerr << "Warning: raising minimum block to " << config.minimum_block << " to fit an ngram in every block." << std::endl;
- }
- UTIL_THROW_IF(config.sort.buffer_size < config.minimum_block, util::Exception, "Sort block size " << config.sort.buffer_size << " is below the minimum block size " << config.minimum_block << ".");
- UTIL_THROW_IF(config.TotalMemory() < config.minimum_block * config.order * config.block_count, util::Exception,
- "Not enough memory to fit " << (config.order * config.block_count) << " blocks with minimum size " << config.minimum_block << ". Increase memory to " << (config.minimum_block * config.order * config.block_count) << " bytes or decrease the minimum block size.");
-
- Master master(config, output.Steps());
- // master's destructor will wait for chains. But they might be deadlocked if
- // this thread dies because e.g. it ran out of memory.
- try {
- VocabNumbering numbering(output.VocabFile(), config.TempPrefix(), config.renumber_vocabulary);
- uint64_t token_count;
- WordIndex type_count;
- std::string text_file_name;
- std::vector<bool> prune_words;
- util::scoped_ptr<util::stream::Sort<SuffixOrder, CombineCounts> > sorted_counts(
- CountText(text_file, numbering.WriteOnTheFly(), master, token_count, type_count, text_file_name, prune_words));
- std::cerr << "Unigram tokens " << token_count << " types " << type_count << std::endl;
-
- // Create vocab mapping, which uses temporary memory, while nothing else is happening.
- std::size_t subtract_for_numbering = numbering.ComputeMapping(type_count);
-
- std::cerr << "=== 2/" << master.Steps() << " Calculating and sorting adjusted counts ===" << std::endl;
- master.InitForAdjust(*sorted_counts, type_count, subtract_for_numbering);
- sorted_counts.reset();
-
- std::vector<uint64_t> counts;
- std::vector<uint64_t> counts_pruned;
- std::vector<Discount> discounts;
- master >> AdjustCounts(config.prune_thresholds, counts, counts_pruned, prune_words, config.discount, discounts);
- numbering.ApplyRenumber(master.MutableChains());
-
- {
- util::FixedArray<util::stream::FileBuffer> gammas;
- Sorts<SuffixOrder> primary;
- InitialProbabilities(counts, counts_pruned, discounts, master, primary, gammas, config.prune_thresholds, config.prune_vocab, numbering.Specials());
- output.SetHeader(HeaderInfo(text_file_name, token_count, counts_pruned));
- // Also does output.
- InterpolateProbabilities(counts_pruned, master, primary, gammas, output, numbering.Specials());
- }
- } catch (const util::Exception &e) {
- std::cerr << e.what() << std::endl;
- abort();
- }
-}
-
-}} // namespaces
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/builder/pipeline.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/builder/pipeline.hh b/ext/kenlm/lm/builder/pipeline.hh
deleted file mode 100644
index 66f1fd9..0000000
--- a/ext/kenlm/lm/builder/pipeline.hh
+++ /dev/null
@@ -1,76 +0,0 @@
-#ifndef LM_BUILDER_PIPELINE_H
-#define LM_BUILDER_PIPELINE_H
-
-#include "lm/builder/adjust_counts.hh"
-#include "lm/builder/initial_probabilities.hh"
-#include "lm/builder/header_info.hh"
-#include "lm/lm_exception.hh"
-#include "lm/word_index.hh"
-#include "util/stream/config.hh"
-#include "util/file_piece.hh"
-
-#include <string>
-#include <cstddef>
-
-namespace lm { namespace builder {
-
-class Output;
-
-struct PipelineConfig {
- std::size_t order;
- util::stream::SortConfig sort;
- InitialProbabilitiesConfig initial_probs;
- util::stream::ChainConfig read_backoffs;
-
- // Estimated vocabulary size. Used for sizing CorpusCount memory and
- // initial probing hash table sizing, also in CorpusCount.
- lm::WordIndex vocab_estimate;
-
- // Minimum block size to tolerate.
- std::size_t minimum_block;
-
- // Number of blocks to use. This will be overridden to 1 if everything fits.
- std::size_t block_count;
-
- // n-gram count thresholds for pruning. 0 values means no pruning for
- // corresponding n-gram order
- std::vector<uint64_t> prune_thresholds; //mjd
- bool prune_vocab;
- std::string prune_vocab_file;
-
- /* Renumber the vocabulary the way the trie likes it? */
- bool renumber_vocabulary;
-
- // What to do with discount failures.
- DiscountConfig discount;
-
- // Compute collapsed q values instead of probability and backoff
- bool output_q;
-
- /* Computing the perplexity of LMs with different vocabularies is hard. For
- * example, the lowest perplexity is attained by a unigram model that
- * predicts p(<unk>) = 1 and has no other vocabulary. Also, linearly
- * interpolated models will sum to more than 1 because <unk> is duplicated
- * (SRI just pretends p(<unk>) = 0 for these purposes, which makes it sum to
- * 1 but comes with its own problems). This option will make the vocabulary
- * a particular size by replicating <unk> multiple times for purposes of
- * computing vocabulary size. It has no effect if the actual vocabulary is
- * larger. This parameter serves the same purpose as IRSTLM's "dub".
- */
- uint64_t vocab_size_for_unk;
-
- /* What to do the first time <s>, </s>, or <unk> appears in the input. If
- * this is anything but THROW_UP, then the symbol will always be treated as
- * whitespace.
- */
- WarningAction disallowed_symbol_action;
-
- const std::string &TempPrefix() const { return sort.temp_prefix; }
- std::size_t TotalMemory() const { return sort.total_memory; }
-};
-
-// Takes ownership of text_file and out_arpa.
-void Pipeline(PipelineConfig &config, int text_file, Output &output);
-
-}} // namespaces
-#endif // LM_BUILDER_PIPELINE_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/common/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/common/CMakeLists.txt b/ext/kenlm/lm/common/CMakeLists.txt
deleted file mode 100644
index 942e24b..0000000
--- a/ext/kenlm/lm/common/CMakeLists.txt
+++ /dev/null
@@ -1,40 +0,0 @@
-cmake_minimum_required(VERSION 2.8.8)
-#
-# The KenLM cmake files make use of add_library(... OBJECTS ...)
-#
-# This syntax allows grouping of source files when compiling
-# (effectively creating "fake" libraries based on source subdirs).
-#
-# This syntax was only added in cmake version 2.8.8
-#
-# see http://www.cmake.org/Wiki/CMake/Tutorials/Object_Library
-
-
-# This CMake file was created by Lane Schwartz <do...@gmail.com>
-
-# Explicitly list the source files for this subdirectory
-#
-# If you add any source files to this subdirectory
-# that should be included in the kenlm library,
-# (this excludes any unit test files)
-# you should add them to the following list:
-#
-# In order to set correct paths to these files
-# in case this variable is referenced by CMake files in the parent directory,
-# we prefix all files with ${CMAKE_CURRENT_SOURCE_DIR}.
-#
-set(KENLM_COMMON_SOURCE
- ${CMAKE_CURRENT_SOURCE_DIR}/model_buffer.cc
- ${CMAKE_CURRENT_SOURCE_DIR}/print.cc
- ${CMAKE_CURRENT_SOURCE_DIR}/renumber.cc
- ${CMAKE_CURRENT_SOURCE_DIR}/size_option.cc
- )
-
-
-# Group these objects together for later use.
-#
-# Given add_library(foo OBJECT ${my_foo_sources}),
-# refer to these objects as $<TARGET_OBJECTS:foo>
-#
-add_library(kenlm_common OBJECT ${KENLM_COMMON_SOURCE})
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/common/Jamfile
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/common/Jamfile b/ext/kenlm/lm/common/Jamfile
deleted file mode 100644
index c9bdfd0..0000000
--- a/ext/kenlm/lm/common/Jamfile
+++ /dev/null
@@ -1,2 +0,0 @@
-fakelib common : [ glob *.cc : *test.cc *main.cc ]
- ../../util//kenutil ../../util/stream//stream ../../util/double-conversion//double-conversion ..//kenlm /top//boost_program_options ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/common/compare.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/common/compare.hh b/ext/kenlm/lm/common/compare.hh
deleted file mode 100644
index 1c7cd24..0000000
--- a/ext/kenlm/lm/common/compare.hh
+++ /dev/null
@@ -1,174 +0,0 @@
-#ifndef LM_COMMON_COMPARE_H
-#define LM_COMMON_COMPARE_H
-
-#include "lm/word_index.hh"
-
-#include <functional>
-#include <string>
-
-namespace lm {
-
-/**
- * Abstract parent class for defining custom n-gram comparators.
- */
-template <class Child> class Comparator : public std::binary_function<const void *, const void *, bool> {
- public:
-
- /**
- * Constructs a comparator capable of comparing two n-grams.
- *
- * @param order Number of words in each n-gram
- */
- explicit Comparator(std::size_t order) : order_(order) {}
-
- /**
- * Applies the comparator using the Compare method that must be defined in any class that inherits from this class.
- *
- * @param lhs A pointer to the n-gram on the left-hand side of the comparison
- * @param rhs A pointer to the n-gram on the right-hand side of the comparison
- *
- * @see ContextOrder::Compare
- * @see PrefixOrder::Compare
- * @see SuffixOrder::Compare
- */
- inline bool operator()(const void *lhs, const void *rhs) const {
- return static_cast<const Child*>(this)->Compare(static_cast<const WordIndex*>(lhs), static_cast<const WordIndex*>(rhs));
- }
-
- /** Gets the n-gram order defined for this comparator. */
- std::size_t Order() const { return order_; }
-
- protected:
- std::size_t order_;
-};
-
-/**
- * N-gram comparator that compares n-grams according to their reverse (suffix) order.
- *
- * This comparator compares n-grams lexicographically, one word at a time,
- * beginning with the last word of each n-gram and ending with the first word of each n-gram.
- *
- * Some examples of n-gram comparisons as defined by this comparator:
- * - a b c == a b c
- * - a b c < a b d
- * - a b c > a d b
- * - a b c > a b b
- * - a b c > x a c
- * - a b c < x y z
- */
-class SuffixOrder : public Comparator<SuffixOrder> {
- public:
-
- /**
- * Constructs a comparator capable of comparing two n-grams.
- *
- * @param order Number of words in each n-gram
- */
- explicit SuffixOrder(std::size_t order) : Comparator<SuffixOrder>(order) {}
-
- /**
- * Compares two n-grams lexicographically, one word at a time,
- * beginning with the last word of each n-gram and ending with the first word of each n-gram.
- *
- * @param lhs A pointer to the n-gram on the left-hand side of the comparison
- * @param rhs A pointer to the n-gram on the right-hand side of the comparison
- */
- inline bool Compare(const WordIndex *lhs, const WordIndex *rhs) const {
- for (std::size_t i = order_ - 1; i != 0; --i) {
- if (lhs[i] != rhs[i])
- return lhs[i] < rhs[i];
- }
- return lhs[0] < rhs[0];
- }
-
- static const unsigned kMatchOffset = 1;
-};
-
-
-/**
- * N-gram comparator that compares n-grams according to the reverse (suffix) order of the n-gram context.
- *
- * This comparator compares n-grams lexicographically, one word at a time,
- * beginning with the penultimate word of each n-gram and ending with the first word of each n-gram;
- * finally, this comparator compares the last word of each n-gram.
- *
- * Some examples of n-gram comparisons as defined by this comparator:
- * - a b c == a b c
- * - a b c < a b d
- * - a b c < a d b
- * - a b c > a b b
- * - a b c > x a c
- * - a b c < x y z
- */
-class ContextOrder : public Comparator<ContextOrder> {
- public:
-
- /**
- * Constructs a comparator capable of comparing two n-grams.
- *
- * @param order Number of words in each n-gram
- */
- explicit ContextOrder(std::size_t order) : Comparator<ContextOrder>(order) {}
-
- /**
- * Compares two n-grams lexicographically, one word at a time,
- * beginning with the penultimate word of each n-gram and ending with the first word of each n-gram;
- * finally, this comparator compares the last word of each n-gram.
- *
- * @param lhs A pointer to the n-gram on the left-hand side of the comparison
- * @param rhs A pointer to the n-gram on the right-hand side of the comparison
- */
- inline bool Compare(const WordIndex *lhs, const WordIndex *rhs) const {
- for (int i = order_ - 2; i >= 0; --i) {
- if (lhs[i] != rhs[i])
- return lhs[i] < rhs[i];
- }
- return lhs[order_ - 1] < rhs[order_ - 1];
- }
-};
-
-/**
- * N-gram comparator that compares n-grams according to their natural (prefix) order.
- *
- * This comparator compares n-grams lexicographically, one word at a time,
- * beginning with the first word of each n-gram and ending with the last word of each n-gram.
- *
- * Some examples of n-gram comparisons as defined by this comparator:
- * - a b c == a b c
- * - a b c < a b d
- * - a b c < a d b
- * - a b c > a b b
- * - a b c < x a c
- * - a b c < x y z
- */
-class PrefixOrder : public Comparator<PrefixOrder> {
- public:
-
- /**
- * Constructs a comparator capable of comparing two n-grams.
- *
- * @param order Number of words in each n-gram
- */
- explicit PrefixOrder(std::size_t order) : Comparator<PrefixOrder>(order) {}
-
- /**
- * Compares two n-grams lexicographically, one word at a time,
- * beginning with the first word of each n-gram and ending with the last word of each n-gram.
- *
- * @param lhs A pointer to the n-gram on the left-hand side of the comparison
- * @param rhs A pointer to the n-gram on the right-hand side of the comparison
- */
- inline bool Compare(const WordIndex *lhs, const WordIndex *rhs) const {
- for (std::size_t i = 0; i < order_; ++i) {
- if (lhs[i] != rhs[i])
- return lhs[i] < rhs[i];
- }
- return false;
- }
-
- static const unsigned kMatchOffset = 0;
-};
-
-} // namespace lm
-
-#endif // LM_COMMON_COMPARE_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/common/joint_order.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/common/joint_order.hh b/ext/kenlm/lm/common/joint_order.hh
deleted file mode 100644
index 6113bb8..0000000
--- a/ext/kenlm/lm/common/joint_order.hh
+++ /dev/null
@@ -1,71 +0,0 @@
-#ifndef LM_COMMON_JOINT_ORDER_H
-#define LM_COMMON_JOINT_ORDER_H
-
-#include "lm/common/ngram_stream.hh"
-#include "lm/lm_exception.hh"
-
-#ifdef DEBUG
-#include "util/fixed_array.hh"
-#include <iostream>
-#endif
-
-#include <cstring>
-
-namespace lm {
-
-template <class Callback, class Compare> void JointOrder(const util::stream::ChainPositions &positions, Callback &callback) {
- // Allow matching to reference streams[-1].
- util::FixedArray<ProxyStream<NGramHeader> > streams_with_dummy(positions.size() + 1);
- // A bogus stream for [-1].
- streams_with_dummy.push_back();
- for (std::size_t i = 0; i < positions.size(); ++i) {
- streams_with_dummy.push_back(positions[i], NGramHeader(NULL, i + 1));
- }
- ProxyStream<NGramHeader> *streams = streams_with_dummy.begin() + 1;
-
- std::size_t order;
- for (order = 0; order < positions.size() && streams[order]; ++order) {}
- assert(order); // should always have <unk>.
-
- // Debugging only: call comparison function to sanity check order.
-#ifdef DEBUG
- util::FixedArray<Compare> less_compare(order);
- for (unsigned i = 0; i < order; ++i)
- less_compare.push_back(i + 1);
-#endif // DEBUG
-
- std::size_t current = 0;
- while (true) {
- // Does the context match the lower one?
- if (!memcmp(streams[static_cast<int>(current) - 1]->begin(), streams[current]->begin() + Compare::kMatchOffset, sizeof(WordIndex) * current)) {
- callback.Enter(current, streams[current].Get());
- // Transition to looking for extensions.
- if (++current < order) continue;
- }
-#ifdef DEBUG
- // match_check[current - 1] matches current-grams
- // The lower-order stream (which skips fewer current-grams) should always be <= the higher order-stream (which can skip current-grams).
- else if (!less_compare[current - 1](streams[static_cast<int>(current) - 1]->begin(), streams[current]->begin() + Compare::kMatchOffset)) {
- std::cerr << "Stream out of order detected" << std::endl;
- abort();
- }
-#endif // DEBUG
- // No extension left.
- while(true) {
- assert(current > 0);
- --current;
- callback.Exit(current, streams[current].Get());
-
- if (++streams[current]) break;
-
- UTIL_THROW_IF(order != current + 1, FormatLoadException, "Detected n-gram without matching suffix");
-
- order = current;
- if (!order) return;
- }
- }
-}
-
-} // namespaces
-
-#endif // LM_COMMON_JOINT_ORDER_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/common/model_buffer.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/common/model_buffer.cc b/ext/kenlm/lm/common/model_buffer.cc
deleted file mode 100644
index ae9b08c..0000000
--- a/ext/kenlm/lm/common/model_buffer.cc
+++ /dev/null
@@ -1,91 +0,0 @@
-#include "lm/common/model_buffer.hh"
-#include "util/exception.hh"
-#include "util/file_stream.hh"
-#include "util/file.hh"
-#include "util/file_piece.hh"
-#include "util/stream/io.hh"
-#include "util/stream/multi_stream.hh"
-
-#include <boost/lexical_cast.hpp>
-
-namespace lm {
-
-namespace {
-const char kMetadataHeader[] = "KenLM intermediate binary file";
-} // namespace
-
-ModelBuffer::ModelBuffer(StringPiece file_base, bool keep_buffer, bool output_q)
- : file_base_(file_base.data(), file_base.size()), keep_buffer_(keep_buffer), output_q_(output_q),
- vocab_file_(keep_buffer ? util::CreateOrThrow((file_base_ + ".vocab").c_str()) : util::MakeTemp(file_base_)) {}
-
-ModelBuffer::ModelBuffer(StringPiece file_base)
- : file_base_(file_base.data(), file_base.size()), keep_buffer_(false) {
- const std::string full_name = file_base_ + ".kenlm_intermediate";
- util::FilePiece in(full_name.c_str());
- StringPiece token = in.ReadLine();
- UTIL_THROW_IF2(token != kMetadataHeader, "File " << full_name << " begins with \"" << token << "\" not " << kMetadataHeader);
-
- token = in.ReadDelimited();
- UTIL_THROW_IF2(token != "Counts", "Expected Counts, got \"" << token << "\" in " << full_name);
- char got;
- while ((got = in.get()) == ' ') {
- counts_.push_back(in.ReadULong());
- }
- UTIL_THROW_IF2(got != '\n', "Expected newline at end of counts.");
-
- token = in.ReadDelimited();
- UTIL_THROW_IF2(token != "Payload", "Expected Payload, got \"" << token << "\" in " << full_name);
- token = in.ReadDelimited();
- if (token == "q") {
- output_q_ = true;
- } else if (token == "pb") {
- output_q_ = false;
- } else {
- UTIL_THROW(util::Exception, "Unknown payload " << token);
- }
-
- vocab_file_.reset(util::OpenReadOrThrow((file_base_ + ".vocab").c_str()));
-
- files_.Init(counts_.size());
- for (unsigned long i = 0; i < counts_.size(); ++i) {
- files_.push_back(util::OpenReadOrThrow((file_base_ + '.' + boost::lexical_cast<std::string>(i + 1)).c_str()));
- }
-}
-
-void ModelBuffer::Sink(util::stream::Chains &chains, const std::vector<uint64_t> &counts) {
- counts_ = counts;
- // Open files.
- files_.Init(chains.size());
- for (std::size_t i = 0; i < chains.size(); ++i) {
- if (keep_buffer_) {
- files_.push_back(util::CreateOrThrow(
- (file_base_ + '.' + boost::lexical_cast<std::string>(i + 1)).c_str()
- ));
- } else {
- files_.push_back(util::MakeTemp(file_base_));
- }
- chains[i] >> util::stream::Write(files_.back().get());
- }
- if (keep_buffer_) {
- util::scoped_fd metadata(util::CreateOrThrow((file_base_ + ".kenlm_intermediate").c_str()));
- util::FileStream meta(metadata.get(), 200);
- meta << kMetadataHeader << "\nCounts";
- for (std::vector<uint64_t>::const_iterator i = counts_.begin(); i != counts_.end(); ++i) {
- meta << ' ' << *i;
- }
- meta << "\nPayload " << (output_q_ ? "q" : "pb") << '\n';
- }
-}
-
-void ModelBuffer::Source(util::stream::Chains &chains) {
- assert(chains.size() <= files_.size());
- for (unsigned int i = 0; i < chains.size(); ++i) {
- chains[i] >> util::stream::PRead(files_[i].get());
- }
-}
-
-void ModelBuffer::Source(std::size_t order_minus_1, util::stream::Chain &chain) {
- chain >> util::stream::PRead(files_[order_minus_1].get());
-}
-
-} // namespace
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/common/model_buffer.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/common/model_buffer.hh b/ext/kenlm/lm/common/model_buffer.hh
deleted file mode 100644
index 92662bb..0000000
--- a/ext/kenlm/lm/common/model_buffer.hh
+++ /dev/null
@@ -1,63 +0,0 @@
-#ifndef LM_COMMON_MODEL_BUFFER_H
-#define LM_COMMON_MODEL_BUFFER_H
-
-/* Format with separate files in suffix order. Each file contains
- * n-grams of the same order.
- */
-
-#include "util/file.hh"
-#include "util/fixed_array.hh"
-
-#include <string>
-#include <vector>
-
-namespace util { namespace stream {
-class Chains;
-class Chain;
-}} // namespaces
-
-namespace lm {
-
-class ModelBuffer {
- public:
- // Construct for writing. Must call VocabFile() and fill it with null-delimited vocab words.
- ModelBuffer(StringPiece file_base, bool keep_buffer, bool output_q);
-
- // Load from file.
- explicit ModelBuffer(StringPiece file_base);
-
- // Must call VocabFile and populate before calling this function.
- void Sink(util::stream::Chains &chains, const std::vector<uint64_t> &counts);
-
- // Read files and write to the given chains. If fewer chains are provided,
- // only do the lower orders.
- void Source(util::stream::Chains &chains);
-
- void Source(std::size_t order_minus_1, util::stream::Chain &chain);
-
- // The order of the n-gram model that is associated with the model buffer.
- std::size_t Order() const { return counts_.size(); }
- // Requires Sink or load from file.
- const std::vector<uint64_t> &Counts() const {
- assert(!counts_.empty());
- return counts_;
- }
-
- int VocabFile() const { return vocab_file_.get(); }
- int StealVocabFile() { return vocab_file_.release(); }
-
- bool Keep() const { return keep_buffer_; }
-
- private:
- const std::string file_base_;
- const bool keep_buffer_;
- bool output_q_;
- std::vector<uint64_t> counts_;
-
- util::scoped_fd vocab_file_;
- util::FixedArray<util::scoped_fd> files_;
-};
-
-} // namespace lm
-
-#endif // LM_COMMON_MODEL_BUFFER_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/common/ngram.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/common/ngram.hh b/ext/kenlm/lm/common/ngram.hh
deleted file mode 100644
index 7a6d1c3..0000000
--- a/ext/kenlm/lm/common/ngram.hh
+++ /dev/null
@@ -1,77 +0,0 @@
-#ifndef LM_COMMON_NGRAM_H
-#define LM_COMMON_NGRAM_H
-
-#include "lm/weights.hh"
-#include "lm/word_index.hh"
-
-#include <cstddef>
-#include <cassert>
-#include <stdint.h>
-#include <cstring>
-
-namespace lm {
-
-class NGramHeader {
- public:
- NGramHeader(void *begin, std::size_t order)
- : begin_(static_cast<WordIndex*>(begin)), end_(begin_ + order) {}
-
- NGramHeader() : begin_(NULL), end_(NULL) {}
-
- const uint8_t *Base() const { return reinterpret_cast<const uint8_t*>(begin_); }
- uint8_t *Base() { return reinterpret_cast<uint8_t*>(begin_); }
-
- void ReBase(void *to) {
- std::size_t difference = end_ - begin_;
- begin_ = reinterpret_cast<WordIndex*>(to);
- end_ = begin_ + difference;
- }
-
- // These are for the vocab index.
- // Lower-case in deference to STL.
- const WordIndex *begin() const { return begin_; }
- WordIndex *begin() { return begin_; }
- const WordIndex *end() const { return end_; }
- WordIndex *end() { return end_; }
-
- std::size_t size() const { return end_ - begin_; }
- std::size_t Order() const { return end_ - begin_; }
-
- private:
- WordIndex *begin_, *end_;
-};
-
-template <class PayloadT> class NGram : public NGramHeader {
- public:
- typedef PayloadT Payload;
-
- NGram() : NGramHeader(NULL, 0) {}
-
- NGram(void *begin, std::size_t order) : NGramHeader(begin, order) {}
-
- // Would do operator++ but that can get confusing for a stream.
- void NextInMemory() {
- ReBase(&Value() + 1);
- }
-
- static std::size_t TotalSize(std::size_t order) {
- return order * sizeof(WordIndex) + sizeof(Payload);
- }
- std::size_t TotalSize() const {
- // Compiler should optimize this.
- return TotalSize(Order());
- }
-
- static std::size_t OrderFromSize(std::size_t size) {
- std::size_t ret = (size - sizeof(Payload)) / sizeof(WordIndex);
- assert(size == TotalSize(ret));
- return ret;
- }
-
- const Payload &Value() const { return *reinterpret_cast<const Payload *>(end()); }
- Payload &Value() { return *reinterpret_cast<Payload *>(end()); }
-};
-
-} // namespace lm
-
-#endif // LM_COMMON_NGRAM_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/common/ngram_stream.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/common/ngram_stream.hh b/ext/kenlm/lm/common/ngram_stream.hh
deleted file mode 100644
index 8bdf36e..0000000
--- a/ext/kenlm/lm/common/ngram_stream.hh
+++ /dev/null
@@ -1,65 +0,0 @@
-#ifndef LM_BUILDER_NGRAM_STREAM_H
-#define LM_BUILDER_NGRAM_STREAM_H
-
-#include "lm/common/ngram.hh"
-#include "util/stream/chain.hh"
-#include "util/stream/multi_stream.hh"
-#include "util/stream/stream.hh"
-
-#include <cstddef>
-
-namespace lm {
-
-template <class Proxy> class ProxyStream {
- public:
- // Make an invalid stream.
- ProxyStream() {}
-
- explicit ProxyStream(const util::stream::ChainPosition &position, const Proxy &proxy = Proxy())
- : proxy_(proxy), stream_(position) {
- proxy_.ReBase(stream_.Get());
- }
-
- Proxy &operator*() { return proxy_; }
- const Proxy &operator*() const { return proxy_; }
-
- Proxy *operator->() { return &proxy_; }
- const Proxy *operator->() const { return &proxy_; }
-
- void *Get() { return stream_.Get(); }
- const void *Get() const { return stream_.Get(); }
-
- operator bool() const { return stream_; }
- bool operator!() const { return !stream_; }
- void Poison() { stream_.Poison(); }
-
- ProxyStream<Proxy> &operator++() {
- ++stream_;
- proxy_.ReBase(stream_.Get());
- return *this;
- }
-
- private:
- Proxy proxy_;
- util::stream::Stream stream_;
-};
-
-template <class Payload> class NGramStream : public ProxyStream<NGram<Payload> > {
- public:
- // Make an invalid stream.
- NGramStream() {}
-
- explicit NGramStream(const util::stream::ChainPosition &position) :
- ProxyStream<NGram<Payload> >(position, NGram<Payload>(NULL, NGram<Payload>::OrderFromSize(position.GetChain().EntrySize()))) {}
-};
-
-template <class Payload> class NGramStreams : public util::stream::GenericStreams<NGramStream<Payload> > {
- private:
- typedef util::stream::GenericStreams<NGramStream<Payload> > P;
- public:
- NGramStreams() : P() {}
- NGramStreams(const util::stream::ChainPositions &positions) : P(positions) {}
-};
-
-} // namespace
-#endif // LM_BUILDER_NGRAM_STREAM_H
[49/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/.gitignore
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/.gitignore b/ext/kenlm/.gitignore
deleted file mode 100644
index 6baae3e..0000000
--- a/ext/kenlm/.gitignore
+++ /dev/null
@@ -1,32 +0,0 @@
-bin/
-lm/bin/
-util/bin/
-util/file_piece.cc.gz
-lib/
-dist
-*.swp
-*.o
-query
-build_binary
-kenlm_max_order
-lm/left_test
-lm/model_test
-util/bit_packing_test
-util/file_piece_test
-util/joint_sort_test
-util/probing_hash_table_test
-util/read_compressed_test
-util/sorted_uniform_test
-previous.sh
-jam-files/bjam-1_55
-jam-files/engine/bin.*/
-jam-files/engine/bootstrap/
-doc/
-._*
-include/
-windows/Win32
-windows/x64
-windows/*.user
-windows/*.sdf
-windows/*.opensdf
-windows/*.suo
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/BUILDING
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/BUILDING b/ext/kenlm/BUILDING
deleted file mode 100644
index e706815..0000000
--- a/ext/kenlm/BUILDING
+++ /dev/null
@@ -1,10 +0,0 @@
-If you have Boost >= 1.36.0 installed:
- ./bjam
- ./bjam install --prefix=/usr #optional
-
-If you only want the query code and do not care about compression (.gz, .bz2, and .xz):
- ./compile_query_only.sh
-
-Windows:
- The windows directory has visual studio files. Note that you need to compile
- the kenlm project before build_binary and ngram_query projects.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/CMakeLists.txt b/ext/kenlm/CMakeLists.txt
deleted file mode 100644
index 15d9931..0000000
--- a/ext/kenlm/CMakeLists.txt
+++ /dev/null
@@ -1,57 +0,0 @@
-cmake_minimum_required(VERSION 2.8.8)
-#
-# The KenLM cmake files make use of add_library(... OBJECTS ...)
-#
-# This syntax allows grouping of source files when compiling
-# (effectively creating "fake" libraries based on source subdirs).
-#
-# This syntax was only added in cmake version 2.8.8
-#
-# see http://www.cmake.org/Wiki/CMake/Tutorials/Object_Library
-
-
-# This CMake file was created by Lane Schwartz <do...@gmail.com>
-
-
-# Define a single cmake project
-project(kenlm)
-
-# Compile all executables into bin/
-set(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/bin)
-
-# Compile all libraries into lib/
-set(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/lib)
-
-# Tell cmake that we want unit tests to be compiled
-include(CTest)
-enable_testing()
-
-# Add our CMake helper functions
-include(cmake/KenLMFunctions.cmake)
-
-# And our helper modules
-list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/modules)
-
-# We need boost
-find_package(Boost 1.36.0 REQUIRED COMPONENTS
- program_options
- system
- thread
- unit_test_framework
-)
-
-
-
-
-# Define where include files live
-include_directories(
- ${PROJECT_SOURCE_DIR}
- ${Boost_INCLUDE_DIRS}
-)
-
-option(BUILD_INTERPOLATE "Build language model interpolation code" OFF)
-
-# Process subdirectories
-add_subdirectory(util)
-add_subdirectory(lm)
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/COPYING
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/COPYING b/ext/kenlm/COPYING
deleted file mode 100644
index 4362b49..0000000
--- a/ext/kenlm/COPYING
+++ /dev/null
@@ -1,502 +0,0 @@
- GNU LESSER GENERAL PUBLIC LICENSE
- Version 2.1, February 1999
-
- Copyright (C) 1991, 1999 Free Software Foundation, Inc.
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-[This is the first released version of the Lesser GPL. It also counts
- as the successor of the GNU Library Public License, version 2, hence
- the version number 2.1.]
-
- Preamble
-
- The licenses for most software are designed to take away your
-freedom to share and change it. By contrast, the GNU General Public
-Licenses are intended to guarantee your freedom to share and change
-free software--to make sure the software is free for all its users.
-
- This license, the Lesser General Public License, applies to some
-specially designated software packages--typically libraries--of the
-Free Software Foundation and other authors who decide to use it. You
-can use it too, but we suggest you first think carefully about whether
-this license or the ordinary General Public License is the better
-strategy to use in any particular case, based on the explanations below.
-
- When we speak of free software, we are referring to freedom of use,
-not price. Our General Public Licenses are designed to make sure that
-you have the freedom to distribute copies of free software (and charge
-for this service if you wish); that you receive source code or can get
-it if you want it; that you can change the software and use pieces of
-it in new free programs; and that you are informed that you can do
-these things.
-
- To protect your rights, we need to make restrictions that forbid
-distributors to deny you these rights or to ask you to surrender these
-rights. These restrictions translate to certain responsibilities for
-you if you distribute copies of the library or if you modify it.
-
- For example, if you distribute copies of the library, whether gratis
-or for a fee, you must give the recipients all the rights that we gave
-you. You must make sure that they, too, receive or can get the source
-code. If you link other code with the library, you must provide
-complete object files to the recipients, so that they can relink them
-with the library after making changes to the library and recompiling
-it. And you must show them these terms so they know their rights.
-
- We protect your rights with a two-step method: (1) we copyright the
-library, and (2) we offer you this license, which gives you legal
-permission to copy, distribute and/or modify the library.
-
- To protect each distributor, we want to make it very clear that
-there is no warranty for the free library. Also, if the library is
-modified by someone else and passed on, the recipients should know
-that what they have is not the original version, so that the original
-author's reputation will not be affected by problems that might be
-introduced by others.
-
- Finally, software patents pose a constant threat to the existence of
-any free program. We wish to make sure that a company cannot
-effectively restrict the users of a free program by obtaining a
-restrictive license from a patent holder. Therefore, we insist that
-any patent license obtained for a version of the library must be
-consistent with the full freedom of use specified in this license.
-
- Most GNU software, including some libraries, is covered by the
-ordinary GNU General Public License. This license, the GNU Lesser
-General Public License, applies to certain designated libraries, and
-is quite different from the ordinary General Public License. We use
-this license for certain libraries in order to permit linking those
-libraries into non-free programs.
-
- When a program is linked with a library, whether statically or using
-a shared library, the combination of the two is legally speaking a
-combined work, a derivative of the original library. The ordinary
-General Public License therefore permits such linking only if the
-entire combination fits its criteria of freedom. The Lesser General
-Public License permits more lax criteria for linking other code with
-the library.
-
- We call this license the "Lesser" General Public License because it
-does Less to protect the user's freedom than the ordinary General
-Public License. It also provides other free software developers Less
-of an advantage over competing non-free programs. These disadvantages
-are the reason we use the ordinary General Public License for many
-libraries. However, the Lesser license provides advantages in certain
-special circumstances.
-
- For example, on rare occasions, there may be a special need to
-encourage the widest possible use of a certain library, so that it becomes
-a de-facto standard. To achieve this, non-free programs must be
-allowed to use the library. A more frequent case is that a free
-library does the same job as widely used non-free libraries. In this
-case, there is little to gain by limiting the free library to free
-software only, so we use the Lesser General Public License.
-
- In other cases, permission to use a particular library in non-free
-programs enables a greater number of people to use a large body of
-free software. For example, permission to use the GNU C Library in
-non-free programs enables many more people to use the whole GNU
-operating system, as well as its variant, the GNU/Linux operating
-system.
-
- Although the Lesser General Public License is Less protective of the
-users' freedom, it does ensure that the user of a program that is
-linked with the Library has the freedom and the wherewithal to run
-that program using a modified version of the Library.
-
- The precise terms and conditions for copying, distribution and
-modification follow. Pay close attention to the difference between a
-"work based on the library" and a "work that uses the library". The
-former contains code derived from the library, whereas the latter must
-be combined with the library in order to run.
-
- GNU LESSER GENERAL PUBLIC LICENSE
- TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
-
- 0. This License Agreement applies to any software library or other
-program which contains a notice placed by the copyright holder or
-other authorized party saying it may be distributed under the terms of
-this Lesser General Public License (also called "this License").
-Each licensee is addressed as "you".
-
- A "library" means a collection of software functions and/or data
-prepared so as to be conveniently linked with application programs
-(which use some of those functions and data) to form executables.
-
- The "Library", below, refers to any such software library or work
-which has been distributed under these terms. A "work based on the
-Library" means either the Library or any derivative work under
-copyright law: that is to say, a work containing the Library or a
-portion of it, either verbatim or with modifications and/or translated
-straightforwardly into another language. (Hereinafter, translation is
-included without limitation in the term "modification".)
-
- "Source code" for a work means the preferred form of the work for
-making modifications to it. For a library, complete source code means
-all the source code for all modules it contains, plus any associated
-interface definition files, plus the scripts used to control compilation
-and installation of the library.
-
- Activities other than copying, distribution and modification are not
-covered by this License; they are outside its scope. The act of
-running a program using the Library is not restricted, and output from
-such a program is covered only if its contents constitute a work based
-on the Library (independent of the use of the Library in a tool for
-writing it). Whether that is true depends on what the Library does
-and what the program that uses the Library does.
-
- 1. You may copy and distribute verbatim copies of the Library's
-complete source code as you receive it, in any medium, provided that
-you conspicuously and appropriately publish on each copy an
-appropriate copyright notice and disclaimer of warranty; keep intact
-all the notices that refer to this License and to the absence of any
-warranty; and distribute a copy of this License along with the
-Library.
-
- You may charge a fee for the physical act of transferring a copy,
-and you may at your option offer warranty protection in exchange for a
-fee.
-
- 2. You may modify your copy or copies of the Library or any portion
-of it, thus forming a work based on the Library, and copy and
-distribute such modifications or work under the terms of Section 1
-above, provided that you also meet all of these conditions:
-
- a) The modified work must itself be a software library.
-
- b) You must cause the files modified to carry prominent notices
- stating that you changed the files and the date of any change.
-
- c) You must cause the whole of the work to be licensed at no
- charge to all third parties under the terms of this License.
-
- d) If a facility in the modified Library refers to a function or a
- table of data to be supplied by an application program that uses
- the facility, other than as an argument passed when the facility
- is invoked, then you must make a good faith effort to ensure that,
- in the event an application does not supply such function or
- table, the facility still operates, and performs whatever part of
- its purpose remains meaningful.
-
- (For example, a function in a library to compute square roots has
- a purpose that is entirely well-defined independent of the
- application. Therefore, Subsection 2d requires that any
- application-supplied function or table used by this function must
- be optional: if the application does not supply it, the square
- root function must still compute square roots.)
-
-These requirements apply to the modified work as a whole. If
-identifiable sections of that work are not derived from the Library,
-and can be reasonably considered independent and separate works in
-themselves, then this License, and its terms, do not apply to those
-sections when you distribute them as separate works. But when you
-distribute the same sections as part of a whole which is a work based
-on the Library, the distribution of the whole must be on the terms of
-this License, whose permissions for other licensees extend to the
-entire whole, and thus to each and every part regardless of who wrote
-it.
-
-Thus, it is not the intent of this section to claim rights or contest
-your rights to work written entirely by you; rather, the intent is to
-exercise the right to control the distribution of derivative or
-collective works based on the Library.
-
-In addition, mere aggregation of another work not based on the Library
-with the Library (or with a work based on the Library) on a volume of
-a storage or distribution medium does not bring the other work under
-the scope of this License.
-
- 3. You may opt to apply the terms of the ordinary GNU General Public
-License instead of this License to a given copy of the Library. To do
-this, you must alter all the notices that refer to this License, so
-that they refer to the ordinary GNU General Public License, version 2,
-instead of to this License. (If a newer version than version 2 of the
-ordinary GNU General Public License has appeared, then you can specify
-that version instead if you wish.) Do not make any other change in
-these notices.
-
- Once this change is made in a given copy, it is irreversible for
-that copy, so the ordinary GNU General Public License applies to all
-subsequent copies and derivative works made from that copy.
-
- This option is useful when you wish to copy part of the code of
-the Library into a program that is not a library.
-
- 4. You may copy and distribute the Library (or a portion or
-derivative of it, under Section 2) in object code or executable form
-under the terms of Sections 1 and 2 above provided that you accompany
-it with the complete corresponding machine-readable source code, which
-must be distributed under the terms of Sections 1 and 2 above on a
-medium customarily used for software interchange.
-
- If distribution of object code is made by offering access to copy
-from a designated place, then offering equivalent access to copy the
-source code from the same place satisfies the requirement to
-distribute the source code, even though third parties are not
-compelled to copy the source along with the object code.
-
- 5. A program that contains no derivative of any portion of the
-Library, but is designed to work with the Library by being compiled or
-linked with it, is called a "work that uses the Library". Such a
-work, in isolation, is not a derivative work of the Library, and
-therefore falls outside the scope of this License.
-
- However, linking a "work that uses the Library" with the Library
-creates an executable that is a derivative of the Library (because it
-contains portions of the Library), rather than a "work that uses the
-library". The executable is therefore covered by this License.
-Section 6 states terms for distribution of such executables.
-
- When a "work that uses the Library" uses material from a header file
-that is part of the Library, the object code for the work may be a
-derivative work of the Library even though the source code is not.
-Whether this is true is especially significant if the work can be
-linked without the Library, or if the work is itself a library. The
-threshold for this to be true is not precisely defined by law.
-
- If such an object file uses only numerical parameters, data
-structure layouts and accessors, and small macros and small inline
-functions (ten lines or less in length), then the use of the object
-file is unrestricted, regardless of whether it is legally a derivative
-work. (Executables containing this object code plus portions of the
-Library will still fall under Section 6.)
-
- Otherwise, if the work is a derivative of the Library, you may
-distribute the object code for the work under the terms of Section 6.
-Any executables containing that work also fall under Section 6,
-whether or not they are linked directly with the Library itself.
-
- 6. As an exception to the Sections above, you may also combine or
-link a "work that uses the Library" with the Library to produce a
-work containing portions of the Library, and distribute that work
-under terms of your choice, provided that the terms permit
-modification of the work for the customer's own use and reverse
-engineering for debugging such modifications.
-
- You must give prominent notice with each copy of the work that the
-Library is used in it and that the Library and its use are covered by
-this License. You must supply a copy of this License. If the work
-during execution displays copyright notices, you must include the
-copyright notice for the Library among them, as well as a reference
-directing the user to the copy of this License. Also, you must do one
-of these things:
-
- a) Accompany the work with the complete corresponding
- machine-readable source code for the Library including whatever
- changes were used in the work (which must be distributed under
- Sections 1 and 2 above); and, if the work is an executable linked
- with the Library, with the complete machine-readable "work that
- uses the Library", as object code and/or source code, so that the
- user can modify the Library and then relink to produce a modified
- executable containing the modified Library. (It is understood
- that the user who changes the contents of definitions files in the
- Library will not necessarily be able to recompile the application
- to use the modified definitions.)
-
- b) Use a suitable shared library mechanism for linking with the
- Library. A suitable mechanism is one that (1) uses at run time a
- copy of the library already present on the user's computer system,
- rather than copying library functions into the executable, and (2)
- will operate properly with a modified version of the library, if
- the user installs one, as long as the modified version is
- interface-compatible with the version that the work was made with.
-
- c) Accompany the work with a written offer, valid for at
- least three years, to give the same user the materials
- specified in Subsection 6a, above, for a charge no more
- than the cost of performing this distribution.
-
- d) If distribution of the work is made by offering access to copy
- from a designated place, offer equivalent access to copy the above
- specified materials from the same place.
-
- e) Verify that the user has already received a copy of these
- materials or that you have already sent this user a copy.
-
- For an executable, the required form of the "work that uses the
-Library" must include any data and utility programs needed for
-reproducing the executable from it. However, as a special exception,
-the materials to be distributed need not include anything that is
-normally distributed (in either source or binary form) with the major
-components (compiler, kernel, and so on) of the operating system on
-which the executable runs, unless that component itself accompanies
-the executable.
-
- It may happen that this requirement contradicts the license
-restrictions of other proprietary libraries that do not normally
-accompany the operating system. Such a contradiction means you cannot
-use both them and the Library together in an executable that you
-distribute.
-
- 7. You may place library facilities that are a work based on the
-Library side-by-side in a single library together with other library
-facilities not covered by this License, and distribute such a combined
-library, provided that the separate distribution of the work based on
-the Library and of the other library facilities is otherwise
-permitted, and provided that you do these two things:
-
- a) Accompany the combined library with a copy of the same work
- based on the Library, uncombined with any other library
- facilities. This must be distributed under the terms of the
- Sections above.
-
- b) Give prominent notice with the combined library of the fact
- that part of it is a work based on the Library, and explaining
- where to find the accompanying uncombined form of the same work.
-
- 8. You may not copy, modify, sublicense, link with, or distribute
-the Library except as expressly provided under this License. Any
-attempt otherwise to copy, modify, sublicense, link with, or
-distribute the Library is void, and will automatically terminate your
-rights under this License. However, parties who have received copies,
-or rights, from you under this License will not have their licenses
-terminated so long as such parties remain in full compliance.
-
- 9. You are not required to accept this License, since you have not
-signed it. However, nothing else grants you permission to modify or
-distribute the Library or its derivative works. These actions are
-prohibited by law if you do not accept this License. Therefore, by
-modifying or distributing the Library (or any work based on the
-Library), you indicate your acceptance of this License to do so, and
-all its terms and conditions for copying, distributing or modifying
-the Library or works based on it.
-
- 10. Each time you redistribute the Library (or any work based on the
-Library), the recipient automatically receives a license from the
-original licensor to copy, distribute, link with or modify the Library
-subject to these terms and conditions. You may not impose any further
-restrictions on the recipients' exercise of the rights granted herein.
-You are not responsible for enforcing compliance by third parties with
-this License.
-
- 11. If, as a consequence of a court judgment or allegation of patent
-infringement or for any other reason (not limited to patent issues),
-conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License. If you cannot
-distribute so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you
-may not distribute the Library at all. For example, if a patent
-license would not permit royalty-free redistribution of the Library by
-all those who receive copies directly or indirectly through you, then
-the only way you could satisfy both it and this License would be to
-refrain entirely from distribution of the Library.
-
-If any portion of this section is held invalid or unenforceable under any
-particular circumstance, the balance of the section is intended to apply,
-and the section as a whole is intended to apply in other circumstances.
-
-It is not the purpose of this section to induce you to infringe any
-patents or other property right claims or to contest validity of any
-such claims; this section has the sole purpose of protecting the
-integrity of the free software distribution system which is
-implemented by public license practices. Many people have made
-generous contributions to the wide range of software distributed
-through that system in reliance on consistent application of that
-system; it is up to the author/donor to decide if he or she is willing
-to distribute software through any other system and a licensee cannot
-impose that choice.
-
-This section is intended to make thoroughly clear what is believed to
-be a consequence of the rest of this License.
-
- 12. If the distribution and/or use of the Library is restricted in
-certain countries either by patents or by copyrighted interfaces, the
-original copyright holder who places the Library under this License may add
-an explicit geographical distribution limitation excluding those countries,
-so that distribution is permitted only in or among countries not thus
-excluded. In such case, this License incorporates the limitation as if
-written in the body of this License.
-
- 13. The Free Software Foundation may publish revised and/or new
-versions of the Lesser General Public License from time to time.
-Such new versions will be similar in spirit to the present version,
-but may differ in detail to address new problems or concerns.
-
-Each version is given a distinguishing version number. If the Library
-specifies a version number of this License which applies to it and
-"any later version", you have the option of following the terms and
-conditions either of that version or of any later version published by
-the Free Software Foundation. If the Library does not specify a
-license version number, you may choose any version ever published by
-the Free Software Foundation.
-
- 14. If you wish to incorporate parts of the Library into other free
-programs whose distribution conditions are incompatible with these,
-write to the author to ask for permission. For software which is
-copyrighted by the Free Software Foundation, write to the Free
-Software Foundation; we sometimes make exceptions for this. Our
-decision will be guided by the two goals of preserving the free status
-of all derivatives of our free software and of promoting the sharing
-and reuse of software generally.
-
- NO WARRANTY
-
- 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
-WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
-EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
-OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
-KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
-LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
-THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
- 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
-WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
-AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
-FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
-CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
-LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
-RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
-FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
-SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
-DAMAGES.
-
- END OF TERMS AND CONDITIONS
-
- How to Apply These Terms to Your New Libraries
-
- If you develop a new library, and you want it to be of the greatest
-possible use to the public, we recommend making it free software that
-everyone can redistribute and change. You can do so by permitting
-redistribution under these terms (or, alternatively, under the terms of the
-ordinary General Public License).
-
- To apply these terms, attach the following notices to the library. It is
-safest to attach them to the start of each source file to most effectively
-convey the exclusion of warranty; and each file should have at least the
-"copyright" line and a pointer to where the full notice is found.
-
- <one line to give the library's name and a brief idea of what it does.>
- Copyright (C) <year> <name of author>
-
- This library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- This library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with this library; if not, write to the Free Software
- Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-Also add information on how to contact you by electronic and paper mail.
-
-You should also get your employer (if you work as a programmer) or your
-school, if any, to sign a "copyright disclaimer" for the library, if
-necessary. Here is a sample; alter the names:
-
- Yoyodyne, Inc., hereby disclaims all copyright interest in the
- library `Frob' (a library for tweaking knobs) written by James Random Hacker.
-
- <signature of Ty Coon>, 1 April 1990
- Ty Coon, President of Vice
-
-That's all there is to it!
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/COPYING.3
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/COPYING.3 b/ext/kenlm/COPYING.3
deleted file mode 100644
index 94a9ed0..0000000
--- a/ext/kenlm/COPYING.3
+++ /dev/null
@@ -1,674 +0,0 @@
- GNU GENERAL PUBLIC LICENSE
- Version 3, 29 June 2007
-
- Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
- Preamble
-
- The GNU General Public License is a free, copyleft license for
-software and other kinds of works.
-
- The licenses for most software and other practical works are designed
-to take away your freedom to share and change the works. By contrast,
-the GNU General Public License is intended to guarantee your freedom to
-share and change all versions of a program--to make sure it remains free
-software for all its users. We, the Free Software Foundation, use the
-GNU General Public License for most of our software; it applies also to
-any other work released this way by its authors. You can apply it to
-your programs, too.
-
- When we speak of free software, we are referring to freedom, not
-price. Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-them if you wish), that you receive source code or can get it if you
-want it, that you can change the software or use pieces of it in new
-free programs, and that you know you can do these things.
-
- To protect your rights, we need to prevent others from denying you
-these rights or asking you to surrender the rights. Therefore, you have
-certain responsibilities if you distribute copies of the software, or if
-you modify it: responsibilities to respect the freedom of others.
-
- For example, if you distribute copies of such a program, whether
-gratis or for a fee, you must pass on to the recipients the same
-freedoms that you received. You must make sure that they, too, receive
-or can get the source code. And you must show them these terms so they
-know their rights.
-
- Developers that use the GNU GPL protect your rights with two steps:
-(1) assert copyright on the software, and (2) offer you this License
-giving you legal permission to copy, distribute and/or modify it.
-
- For the developers' and authors' protection, the GPL clearly explains
-that there is no warranty for this free software. For both users' and
-authors' sake, the GPL requires that modified versions be marked as
-changed, so that their problems will not be attributed erroneously to
-authors of previous versions.
-
- Some devices are designed to deny users access to install or run
-modified versions of the software inside them, although the manufacturer
-can do so. This is fundamentally incompatible with the aim of
-protecting users' freedom to change the software. The systematic
-pattern of such abuse occurs in the area of products for individuals to
-use, which is precisely where it is most unacceptable. Therefore, we
-have designed this version of the GPL to prohibit the practice for those
-products. If such problems arise substantially in other domains, we
-stand ready to extend this provision to those domains in future versions
-of the GPL, as needed to protect the freedom of users.
-
- Finally, every program is threatened constantly by software patents.
-States should not allow patents to restrict development and use of
-software on general-purpose computers, but in those that do, we wish to
-avoid the special danger that patents applied to a free program could
-make it effectively proprietary. To prevent this, the GPL assures that
-patents cannot be used to render the program non-free.
-
- The precise terms and conditions for copying, distribution and
-modification follow.
-
- TERMS AND CONDITIONS
-
- 0. Definitions.
-
- "This License" refers to version 3 of the GNU General Public License.
-
- "Copyright" also means copyright-like laws that apply to other kinds of
-works, such as semiconductor masks.
-
- "The Program" refers to any copyrightable work licensed under this
-License. Each licensee is addressed as "you". "Licensees" and
-"recipients" may be individuals or organizations.
-
- To "modify" a work means to copy from or adapt all or part of the work
-in a fashion requiring copyright permission, other than the making of an
-exact copy. The resulting work is called a "modified version" of the
-earlier work or a work "based on" the earlier work.
-
- A "covered work" means either the unmodified Program or a work based
-on the Program.
-
- To "propagate" a work means to do anything with it that, without
-permission, would make you directly or secondarily liable for
-infringement under applicable copyright law, except executing it on a
-computer or modifying a private copy. Propagation includes copying,
-distribution (with or without modification), making available to the
-public, and in some countries other activities as well.
-
- To "convey" a work means any kind of propagation that enables other
-parties to make or receive copies. Mere interaction with a user through
-a computer network, with no transfer of a copy, is not conveying.
-
- An interactive user interface displays "Appropriate Legal Notices"
-to the extent that it includes a convenient and prominently visible
-feature that (1) displays an appropriate copyright notice, and (2)
-tells the user that there is no warranty for the work (except to the
-extent that warranties are provided), that licensees may convey the
-work under this License, and how to view a copy of this License. If
-the interface presents a list of user commands or options, such as a
-menu, a prominent item in the list meets this criterion.
-
- 1. Source Code.
-
- The "source code" for a work means the preferred form of the work
-for making modifications to it. "Object code" means any non-source
-form of a work.
-
- A "Standard Interface" means an interface that either is an official
-standard defined by a recognized standards body, or, in the case of
-interfaces specified for a particular programming language, one that
-is widely used among developers working in that language.
-
- The "System Libraries" of an executable work include anything, other
-than the work as a whole, that (a) is included in the normal form of
-packaging a Major Component, but which is not part of that Major
-Component, and (b) serves only to enable use of the work with that
-Major Component, or to implement a Standard Interface for which an
-implementation is available to the public in source code form. A
-"Major Component", in this context, means a major essential component
-(kernel, window system, and so on) of the specific operating system
-(if any) on which the executable work runs, or a compiler used to
-produce the work, or an object code interpreter used to run it.
-
- The "Corresponding Source" for a work in object code form means all
-the source code needed to generate, install, and (for an executable
-work) run the object code and to modify the work, including scripts to
-control those activities. However, it does not include the work's
-System Libraries, or general-purpose tools or generally available free
-programs which are used unmodified in performing those activities but
-which are not part of the work. For example, Corresponding Source
-includes interface definition files associated with source files for
-the work, and the source code for shared libraries and dynamically
-linked subprograms that the work is specifically designed to require,
-such as by intimate data communication or control flow between those
-subprograms and other parts of the work.
-
- The Corresponding Source need not include anything that users
-can regenerate automatically from other parts of the Corresponding
-Source.
-
- The Corresponding Source for a work in source code form is that
-same work.
-
- 2. Basic Permissions.
-
- All rights granted under this License are granted for the term of
-copyright on the Program, and are irrevocable provided the stated
-conditions are met. This License explicitly affirms your unlimited
-permission to run the unmodified Program. The output from running a
-covered work is covered by this License only if the output, given its
-content, constitutes a covered work. This License acknowledges your
-rights of fair use or other equivalent, as provided by copyright law.
-
- You may make, run and propagate covered works that you do not
-convey, without conditions so long as your license otherwise remains
-in force. You may convey covered works to others for the sole purpose
-of having them make modifications exclusively for you, or provide you
-with facilities for running those works, provided that you comply with
-the terms of this License in conveying all material for which you do
-not control copyright. Those thus making or running the covered works
-for you must do so exclusively on your behalf, under your direction
-and control, on terms that prohibit them from making any copies of
-your copyrighted material outside their relationship with you.
-
- Conveying under any other circumstances is permitted solely under
-the conditions stated below. Sublicensing is not allowed; section 10
-makes it unnecessary.
-
- 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
-
- No covered work shall be deemed part of an effective technological
-measure under any applicable law fulfilling obligations under article
-11 of the WIPO copyright treaty adopted on 20 December 1996, or
-similar laws prohibiting or restricting circumvention of such
-measures.
-
- When you convey a covered work, you waive any legal power to forbid
-circumvention of technological measures to the extent such circumvention
-is effected by exercising rights under this License with respect to
-the covered work, and you disclaim any intention to limit operation or
-modification of the work as a means of enforcing, against the work's
-users, your or third parties' legal rights to forbid circumvention of
-technological measures.
-
- 4. Conveying Verbatim Copies.
-
- You may convey verbatim copies of the Program's source code as you
-receive it, in any medium, provided that you conspicuously and
-appropriately publish on each copy an appropriate copyright notice;
-keep intact all notices stating that this License and any
-non-permissive terms added in accord with section 7 apply to the code;
-keep intact all notices of the absence of any warranty; and give all
-recipients a copy of this License along with the Program.
-
- You may charge any price or no price for each copy that you convey,
-and you may offer support or warranty protection for a fee.
-
- 5. Conveying Modified Source Versions.
-
- You may convey a work based on the Program, or the modifications to
-produce it from the Program, in the form of source code under the
-terms of section 4, provided that you also meet all of these conditions:
-
- a) The work must carry prominent notices stating that you modified
- it, and giving a relevant date.
-
- b) The work must carry prominent notices stating that it is
- released under this License and any conditions added under section
- 7. This requirement modifies the requirement in section 4 to
- "keep intact all notices".
-
- c) You must license the entire work, as a whole, under this
- License to anyone who comes into possession of a copy. This
- License will therefore apply, along with any applicable section 7
- additional terms, to the whole of the work, and all its parts,
- regardless of how they are packaged. This License gives no
- permission to license the work in any other way, but it does not
- invalidate such permission if you have separately received it.
-
- d) If the work has interactive user interfaces, each must display
- Appropriate Legal Notices; however, if the Program has interactive
- interfaces that do not display Appropriate Legal Notices, your
- work need not make them do so.
-
- A compilation of a covered work with other separate and independent
-works, which are not by their nature extensions of the covered work,
-and which are not combined with it such as to form a larger program,
-in or on a volume of a storage or distribution medium, is called an
-"aggregate" if the compilation and its resulting copyright are not
-used to limit the access or legal rights of the compilation's users
-beyond what the individual works permit. Inclusion of a covered work
-in an aggregate does not cause this License to apply to the other
-parts of the aggregate.
-
- 6. Conveying Non-Source Forms.
-
- You may convey a covered work in object code form under the terms
-of sections 4 and 5, provided that you also convey the
-machine-readable Corresponding Source under the terms of this License,
-in one of these ways:
-
- a) Convey the object code in, or embodied in, a physical product
- (including a physical distribution medium), accompanied by the
- Corresponding Source fixed on a durable physical medium
- customarily used for software interchange.
-
- b) Convey the object code in, or embodied in, a physical product
- (including a physical distribution medium), accompanied by a
- written offer, valid for at least three years and valid for as
- long as you offer spare parts or customer support for that product
- model, to give anyone who possesses the object code either (1) a
- copy of the Corresponding Source for all the software in the
- product that is covered by this License, on a durable physical
- medium customarily used for software interchange, for a price no
- more than your reasonable cost of physically performing this
- conveying of source, or (2) access to copy the
- Corresponding Source from a network server at no charge.
-
- c) Convey individual copies of the object code with a copy of the
- written offer to provide the Corresponding Source. This
- alternative is allowed only occasionally and noncommercially, and
- only if you received the object code with such an offer, in accord
- with subsection 6b.
-
- d) Convey the object code by offering access from a designated
- place (gratis or for a charge), and offer equivalent access to the
- Corresponding Source in the same way through the same place at no
- further charge. You need not require recipients to copy the
- Corresponding Source along with the object code. If the place to
- copy the object code is a network server, the Corresponding Source
- may be on a different server (operated by you or a third party)
- that supports equivalent copying facilities, provided you maintain
- clear directions next to the object code saying where to find the
- Corresponding Source. Regardless of what server hosts the
- Corresponding Source, you remain obligated to ensure that it is
- available for as long as needed to satisfy these requirements.
-
- e) Convey the object code using peer-to-peer transmission, provided
- you inform other peers where the object code and Corresponding
- Source of the work are being offered to the general public at no
- charge under subsection 6d.
-
- A separable portion of the object code, whose source code is excluded
-from the Corresponding Source as a System Library, need not be
-included in conveying the object code work.
-
- A "User Product" is either (1) a "consumer product", which means any
-tangible personal property which is normally used for personal, family,
-or household purposes, or (2) anything designed or sold for incorporation
-into a dwelling. In determining whether a product is a consumer product,
-doubtful cases shall be resolved in favor of coverage. For a particular
-product received by a particular user, "normally used" refers to a
-typical or common use of that class of product, regardless of the status
-of the particular user or of the way in which the particular user
-actually uses, or expects or is expected to use, the product. A product
-is a consumer product regardless of whether the product has substantial
-commercial, industrial or non-consumer uses, unless such uses represent
-the only significant mode of use of the product.
-
- "Installation Information" for a User Product means any methods,
-procedures, authorization keys, or other information required to install
-and execute modified versions of a covered work in that User Product from
-a modified version of its Corresponding Source. The information must
-suffice to ensure that the continued functioning of the modified object
-code is in no case prevented or interfered with solely because
-modification has been made.
-
- If you convey an object code work under this section in, or with, or
-specifically for use in, a User Product, and the conveying occurs as
-part of a transaction in which the right of possession and use of the
-User Product is transferred to the recipient in perpetuity or for a
-fixed term (regardless of how the transaction is characterized), the
-Corresponding Source conveyed under this section must be accompanied
-by the Installation Information. But this requirement does not apply
-if neither you nor any third party retains the ability to install
-modified object code on the User Product (for example, the work has
-been installed in ROM).
-
- The requirement to provide Installation Information does not include a
-requirement to continue to provide support service, warranty, or updates
-for a work that has been modified or installed by the recipient, or for
-the User Product in which it has been modified or installed. Access to a
-network may be denied when the modification itself materially and
-adversely affects the operation of the network or violates the rules and
-protocols for communication across the network.
-
- Corresponding Source conveyed, and Installation Information provided,
-in accord with this section must be in a format that is publicly
-documented (and with an implementation available to the public in
-source code form), and must require no special password or key for
-unpacking, reading or copying.
-
- 7. Additional Terms.
-
- "Additional permissions" are terms that supplement the terms of this
-License by making exceptions from one or more of its conditions.
-Additional permissions that are applicable to the entire Program shall
-be treated as though they were included in this License, to the extent
-that they are valid under applicable law. If additional permissions
-apply only to part of the Program, that part may be used separately
-under those permissions, but the entire Program remains governed by
-this License without regard to the additional permissions.
-
- When you convey a copy of a covered work, you may at your option
-remove any additional permissions from that copy, or from any part of
-it. (Additional permissions may be written to require their own
-removal in certain cases when you modify the work.) You may place
-additional permissions on material, added by you to a covered work,
-for which you have or can give appropriate copyright permission.
-
- Notwithstanding any other provision of this License, for material you
-add to a covered work, you may (if authorized by the copyright holders of
-that material) supplement the terms of this License with terms:
-
- a) Disclaiming warranty or limiting liability differently from the
- terms of sections 15 and 16 of this License; or
-
- b) Requiring preservation of specified reasonable legal notices or
- author attributions in that material or in the Appropriate Legal
- Notices displayed by works containing it; or
-
- c) Prohibiting misrepresentation of the origin of that material, or
- requiring that modified versions of such material be marked in
- reasonable ways as different from the original version; or
-
- d) Limiting the use for publicity purposes of names of licensors or
- authors of the material; or
-
- e) Declining to grant rights under trademark law for use of some
- trade names, trademarks, or service marks; or
-
- f) Requiring indemnification of licensors and authors of that
- material by anyone who conveys the material (or modified versions of
- it) with contractual assumptions of liability to the recipient, for
- any liability that these contractual assumptions directly impose on
- those licensors and authors.
-
- All other non-permissive additional terms are considered "further
-restrictions" within the meaning of section 10. If the Program as you
-received it, or any part of it, contains a notice stating that it is
-governed by this License along with a term that is a further
-restriction, you may remove that term. If a license document contains
-a further restriction but permits relicensing or conveying under this
-License, you may add to a covered work material governed by the terms
-of that license document, provided that the further restriction does
-not survive such relicensing or conveying.
-
- If you add terms to a covered work in accord with this section, you
-must place, in the relevant source files, a statement of the
-additional terms that apply to those files, or a notice indicating
-where to find the applicable terms.
-
- Additional terms, permissive or non-permissive, may be stated in the
-form of a separately written license, or stated as exceptions;
-the above requirements apply either way.
-
- 8. Termination.
-
- You may not propagate or modify a covered work except as expressly
-provided under this License. Any attempt otherwise to propagate or
-modify it is void, and will automatically terminate your rights under
-this License (including any patent licenses granted under the third
-paragraph of section 11).
-
- However, if you cease all violation of this License, then your
-license from a particular copyright holder is reinstated (a)
-provisionally, unless and until the copyright holder explicitly and
-finally terminates your license, and (b) permanently, if the copyright
-holder fails to notify you of the violation by some reasonable means
-prior to 60 days after the cessation.
-
- Moreover, your license from a particular copyright holder is
-reinstated permanently if the copyright holder notifies you of the
-violation by some reasonable means, this is the first time you have
-received notice of violation of this License (for any work) from that
-copyright holder, and you cure the violation prior to 30 days after
-your receipt of the notice.
-
- Termination of your rights under this section does not terminate the
-licenses of parties who have received copies or rights from you under
-this License. If your rights have been terminated and not permanently
-reinstated, you do not qualify to receive new licenses for the same
-material under section 10.
-
- 9. Acceptance Not Required for Having Copies.
-
- You are not required to accept this License in order to receive or
-run a copy of the Program. Ancillary propagation of a covered work
-occurring solely as a consequence of using peer-to-peer transmission
-to receive a copy likewise does not require acceptance. However,
-nothing other than this License grants you permission to propagate or
-modify any covered work. These actions infringe copyright if you do
-not accept this License. Therefore, by modifying or propagating a
-covered work, you indicate your acceptance of this License to do so.
-
- 10. Automatic Licensing of Downstream Recipients.
-
- Each time you convey a covered work, the recipient automatically
-receives a license from the original licensors, to run, modify and
-propagate that work, subject to this License. You are not responsible
-for enforcing compliance by third parties with this License.
-
- An "entity transaction" is a transaction transferring control of an
-organization, or substantially all assets of one, or subdividing an
-organization, or merging organizations. If propagation of a covered
-work results from an entity transaction, each party to that
-transaction who receives a copy of the work also receives whatever
-licenses to the work the party's predecessor in interest had or could
-give under the previous paragraph, plus a right to possession of the
-Corresponding Source of the work from the predecessor in interest, if
-the predecessor has it or can get it with reasonable efforts.
-
- You may not impose any further restrictions on the exercise of the
-rights granted or affirmed under this License. For example, you may
-not impose a license fee, royalty, or other charge for exercise of
-rights granted under this License, and you may not initiate litigation
-(including a cross-claim or counterclaim in a lawsuit) alleging that
-any patent claim is infringed by making, using, selling, offering for
-sale, or importing the Program or any portion of it.
-
- 11. Patents.
-
- A "contributor" is a copyright holder who authorizes use under this
-License of the Program or a work on which the Program is based. The
-work thus licensed is called the contributor's "contributor version".
-
- A contributor's "essential patent claims" are all patent claims
-owned or controlled by the contributor, whether already acquired or
-hereafter acquired, that would be infringed by some manner, permitted
-by this License, of making, using, or selling its contributor version,
-but do not include claims that would be infringed only as a
-consequence of further modification of the contributor version. For
-purposes of this definition, "control" includes the right to grant
-patent sublicenses in a manner consistent with the requirements of
-this License.
-
- Each contributor grants you a non-exclusive, worldwide, royalty-free
-patent license under the contributor's essential patent claims, to
-make, use, sell, offer for sale, import and otherwise run, modify and
-propagate the contents of its contributor version.
-
- In the following three paragraphs, a "patent license" is any express
-agreement or commitment, however denominated, not to enforce a patent
-(such as an express permission to practice a patent or covenant not to
-sue for patent infringement). To "grant" such a patent license to a
-party means to make such an agreement or commitment not to enforce a
-patent against the party.
-
- If you convey a covered work, knowingly relying on a patent license,
-and the Corresponding Source of the work is not available for anyone
-to copy, free of charge and under the terms of this License, through a
-publicly available network server or other readily accessible means,
-then you must either (1) cause the Corresponding Source to be so
-available, or (2) arrange to deprive yourself of the benefit of the
-patent license for this particular work, or (3) arrange, in a manner
-consistent with the requirements of this License, to extend the patent
-license to downstream recipients. "Knowingly relying" means you have
-actual knowledge that, but for the patent license, your conveying the
-covered work in a country, or your recipient's use of the covered work
-in a country, would infringe one or more identifiable patents in that
-country that you have reason to believe are valid.
-
- If, pursuant to or in connection with a single transaction or
-arrangement, you convey, or propagate by procuring conveyance of, a
-covered work, and grant a patent license to some of the parties
-receiving the covered work authorizing them to use, propagate, modify
-or convey a specific copy of the covered work, then the patent license
-you grant is automatically extended to all recipients of the covered
-work and works based on it.
-
- A patent license is "discriminatory" if it does not include within
-the scope of its coverage, prohibits the exercise of, or is
-conditioned on the non-exercise of one or more of the rights that are
-specifically granted under this License. You may not convey a covered
-work if you are a party to an arrangement with a third party that is
-in the business of distributing software, under which you make payment
-to the third party based on the extent of your activity of conveying
-the work, and under which the third party grants, to any of the
-parties who would receive the covered work from you, a discriminatory
-patent license (a) in connection with copies of the covered work
-conveyed by you (or copies made from those copies), or (b) primarily
-for and in connection with specific products or compilations that
-contain the covered work, unless you entered into that arrangement,
-or that patent license was granted, prior to 28 March 2007.
-
- Nothing in this License shall be construed as excluding or limiting
-any implied license or other defenses to infringement that may
-otherwise be available to you under applicable patent law.
-
- 12. No Surrender of Others' Freedom.
-
- If conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License. If you cannot convey a
-covered work so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you may
-not convey it at all. For example, if you agree to terms that obligate you
-to collect a royalty for further conveying from those to whom you convey
-the Program, the only way you could satisfy both those terms and this
-License would be to refrain entirely from conveying the Program.
-
- 13. Use with the GNU Affero General Public License.
-
- Notwithstanding any other provision of this License, you have
-permission to link or combine any covered work with a work licensed
-under version 3 of the GNU Affero General Public License into a single
-combined work, and to convey the resulting work. The terms of this
-License will continue to apply to the part which is the covered work,
-but the special requirements of the GNU Affero General Public License,
-section 13, concerning interaction through a network will apply to the
-combination as such.
-
- 14. Revised Versions of this License.
-
- The Free Software Foundation may publish revised and/or new versions of
-the GNU General Public License from time to time. Such new versions will
-be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
- Each version is given a distinguishing version number. If the
-Program specifies that a certain numbered version of the GNU General
-Public License "or any later version" applies to it, you have the
-option of following the terms and conditions either of that numbered
-version or of any later version published by the Free Software
-Foundation. If the Program does not specify a version number of the
-GNU General Public License, you may choose any version ever published
-by the Free Software Foundation.
-
- If the Program specifies that a proxy can decide which future
-versions of the GNU General Public License can be used, that proxy's
-public statement of acceptance of a version permanently authorizes you
-to choose that version for the Program.
-
- Later license versions may give you additional or different
-permissions. However, no additional obligations are imposed on any
-author or copyright holder as a result of your choosing to follow a
-later version.
-
- 15. Disclaimer of Warranty.
-
- THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
-APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
-HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
-OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
-THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
-IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
-ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
- 16. Limitation of Liability.
-
- IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
-THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
-GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
-USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
-DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
-PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
-EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGES.
-
- 17. Interpretation of Sections 15 and 16.
-
- If the disclaimer of warranty and limitation of liability provided
-above cannot be given local legal effect according to their terms,
-reviewing courts shall apply local law that most closely approximates
-an absolute waiver of all civil liability in connection with the
-Program, unless a warranty or assumption of liability accompanies a
-copy of the Program in return for a fee.
-
- END OF TERMS AND CONDITIONS
-
- How to Apply These Terms to Your New Programs
-
- If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
- To do so, attach the following notices to the program. It is safest
-to attach them to the start of each source file to most effectively
-state the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
- <one line to give the program's name and a brief idea of what it does.>
- Copyright (C) <year> <name of author>
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Also add information on how to contact you by electronic and paper mail.
-
- If the program does terminal interaction, make it output a short
-notice like this when it starts in an interactive mode:
-
- <program> Copyright (C) <year> <name of author>
- This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
- This is free software, and you are welcome to redistribute it
- under certain conditions; type `show c' for details.
-
-The hypothetical commands `show w' and `show c' should show the appropriate
-parts of the General Public License. Of course, your program's commands
-might be different; for a GUI interface, you would use an "about box".
-
- You should also get your employer (if you work as a programmer) or school,
-if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU GPL, see
-<http://www.gnu.org/licenses/>.
-
- The GNU General Public License does not permit incorporating your program
-into proprietary programs. If your program is a subroutine library, you
-may consider it more useful to permit linking proprietary applications with
-the library. If this is what you want to do, use the GNU Lesser General
-Public License instead of this License. But first, please read
-<http://www.gnu.org/philosophy/why-not-lgpl.html>.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/COPYING.LESSER.3
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/COPYING.LESSER.3 b/ext/kenlm/COPYING.LESSER.3
deleted file mode 100644
index cca7fc2..0000000
--- a/ext/kenlm/COPYING.LESSER.3
+++ /dev/null
@@ -1,165 +0,0 @@
- GNU LESSER GENERAL PUBLIC LICENSE
- Version 3, 29 June 2007
-
- Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-
- This version of the GNU Lesser General Public License incorporates
-the terms and conditions of version 3 of the GNU General Public
-License, supplemented by the additional permissions listed below.
-
- 0. Additional Definitions.
-
- As used herein, "this License" refers to version 3 of the GNU Lesser
-General Public License, and the "GNU GPL" refers to version 3 of the GNU
-General Public License.
-
- "The Library" refers to a covered work governed by this License,
-other than an Application or a Combined Work as defined below.
-
- An "Application" is any work that makes use of an interface provided
-by the Library, but which is not otherwise based on the Library.
-Defining a subclass of a class defined by the Library is deemed a mode
-of using an interface provided by the Library.
-
- A "Combined Work" is a work produced by combining or linking an
-Application with the Library. The particular version of the Library
-with which the Combined Work was made is also called the "Linked
-Version".
-
- The "Minimal Corresponding Source" for a Combined Work means the
-Corresponding Source for the Combined Work, excluding any source code
-for portions of the Combined Work that, considered in isolation, are
-based on the Application, and not on the Linked Version.
-
- The "Corresponding Application Code" for a Combined Work means the
-object code and/or source code for the Application, including any data
-and utility programs needed for reproducing the Combined Work from the
-Application, but excluding the System Libraries of the Combined Work.
-
- 1. Exception to Section 3 of the GNU GPL.
-
- You may convey a covered work under sections 3 and 4 of this License
-without being bound by section 3 of the GNU GPL.
-
- 2. Conveying Modified Versions.
-
- If you modify a copy of the Library, and, in your modifications, a
-facility refers to a function or data to be supplied by an Application
-that uses the facility (other than as an argument passed when the
-facility is invoked), then you may convey a copy of the modified
-version:
-
- a) under this License, provided that you make a good faith effort to
- ensure that, in the event an Application does not supply the
- function or data, the facility still operates, and performs
- whatever part of its purpose remains meaningful, or
-
- b) under the GNU GPL, with none of the additional permissions of
- this License applicable to that copy.
-
- 3. Object Code Incorporating Material from Library Header Files.
-
- The object code form of an Application may incorporate material from
-a header file that is part of the Library. You may convey such object
-code under terms of your choice, provided that, if the incorporated
-material is not limited to numerical parameters, data structure
-layouts and accessors, or small macros, inline functions and templates
-(ten or fewer lines in length), you do both of the following:
-
- a) Give prominent notice with each copy of the object code that the
- Library is used in it and that the Library and its use are
- covered by this License.
-
- b) Accompany the object code with a copy of the GNU GPL and this license
- document.
-
- 4. Combined Works.
-
- You may convey a Combined Work under terms of your choice that,
-taken together, effectively do not restrict modification of the
-portions of the Library contained in the Combined Work and reverse
-engineering for debugging such modifications, if you also do each of
-the following:
-
- a) Give prominent notice with each copy of the Combined Work that
- the Library is used in it and that the Library and its use are
- covered by this License.
-
- b) Accompany the Combined Work with a copy of the GNU GPL and this license
- document.
-
- c) For a Combined Work that displays copyright notices during
- execution, include the copyright notice for the Library among
- these notices, as well as a reference directing the user to the
- copies of the GNU GPL and this license document.
-
- d) Do one of the following:
-
- 0) Convey the Minimal Corresponding Source under the terms of this
- License, and the Corresponding Application Code in a form
- suitable for, and under terms that permit, the user to
- recombine or relink the Application with a modified version of
- the Linked Version to produce a modified Combined Work, in the
- manner specified by section 6 of the GNU GPL for conveying
- Corresponding Source.
-
- 1) Use a suitable shared library mechanism for linking with the
- Library. A suitable mechanism is one that (a) uses at run time
- a copy of the Library already present on the user's computer
- system, and (b) will operate properly with a modified version
- of the Library that is interface-compatible with the Linked
- Version.
-
- e) Provide Installation Information, but only if you would otherwise
- be required to provide such information under section 6 of the
- GNU GPL, and only to the extent that such information is
- necessary to install and execute a modified version of the
- Combined Work produced by recombining or relinking the
- Application with a modified version of the Linked Version. (If
- you use option 4d0, the Installation Information must accompany
- the Minimal Corresponding Source and Corresponding Application
- Code. If you use option 4d1, you must provide the Installation
- Information in the manner specified by section 6 of the GNU GPL
- for conveying Corresponding Source.)
-
- 5. Combined Libraries.
-
- You may place library facilities that are a work based on the
-Library side by side in a single library together with other library
-facilities that are not Applications and are not covered by this
-License, and convey such a combined library under terms of your
-choice, if you do both of the following:
-
- a) Accompany the combined library with a copy of the same work based
- on the Library, uncombined with any other library facilities,
- conveyed under the terms of this License.
-
- b) Give prominent notice with the combined library that part of it
- is a work based on the Library, and explaining where to find the
- accompanying uncombined form of the same work.
-
- 6. Revised Versions of the GNU Lesser General Public License.
-
- The Free Software Foundation may publish revised and/or new versions
-of the GNU Lesser General Public License from time to time. Such new
-versions will be similar in spirit to the present version, but may
-differ in detail to address new problems or concerns.
-
- Each version is given a distinguishing version number. If the
-Library as you received it specifies that a certain numbered version
-of the GNU Lesser General Public License "or any later version"
-applies to it, you have the option of following the terms and
-conditions either of that published version or of any later version
-published by the Free Software Foundation. If the Library as you
-received it does not specify a version number of the GNU Lesser
-General Public License, you may choose any version of the GNU Lesser
-General Public License ever published by the Free Software Foundation.
-
- If the Library as you received it specifies that a proxy can decide
-whether future versions of the GNU Lesser General Public License shall
-apply, that proxy's public statement of acceptance of any version is
-permanent authorization for you to choose that version for the
-Library.
[20/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/jamgram.yy
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/jamgram.yy b/ext/kenlm/jam-files/engine/jamgram.yy
deleted file mode 100644
index 8d20e38..0000000
--- a/ext/kenlm/jam-files/engine/jamgram.yy
+++ /dev/null
@@ -1,329 +0,0 @@
-/*
- * Copyright 1993, 2000 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * jamgram.yy - jam grammar
- *
- * 04/13/94 (seiwald) - added shorthand L0 for null list pointer
- * 06/01/94 (seiwald) - new 'actions existing' does existing sources
- * 08/23/94 (seiwald) - Support for '+=' (append to variable)
- * 08/31/94 (seiwald) - Allow ?= as alias for "default =".
- * 09/15/94 (seiwald) - if conditionals take only single arguments, so
- * that 'if foo == bar' gives syntax error (use =).
- * 02/11/95 (seiwald) - when scanning arguments to rules, only treat
- * punctuation keywords as keywords. All arg lists
- * are terminated with punctuation keywords.
- *
- * 09/11/00 (seiwald) - Support for function calls:
- *
- * Rules now return lists (LIST *), rather than void.
- *
- * New "[ rule ]" syntax evals rule into a LIST.
- *
- * Lists are now generated by compile_list() and
- * compile_append(), and any other rule that indirectly
- * makes a list, rather than being built directly here,
- * so that lists values can contain rule evaluations.
- *
- * New 'return' rule sets the return value, though
- * other statements also may have return values.
- *
- * 'run' production split from 'block' production so
- * that empty blocks can be handled separately.
- */
-
-%token ARG STRING
-
-%left `||` `|`
-%left `&&` `&`
-%left `=` `!=` `in`
-%left `<` `<=` `>` `>=`
-%left `!`
-
-%{
-#include "jam.h"
-
-#include "lists.h"
-#include "parse.h"
-#include "scan.h"
-#include "compile.h"
-#include "object.h"
-#include "rules.h"
-
-# define YYMAXDEPTH 10000 /* for OSF and other less endowed yaccs */
-
-# define F0 -1
-# define P0 (PARSE *)0
-# define S0 (OBJECT *)0
-
-# define pappend( l,r ) parse_make( PARSE_APPEND,l,r,P0,S0,S0,0 )
-# define peval( c,l,r ) parse_make( PARSE_EVAL,l,r,P0,S0,S0,c )
-# define pfor( s,l,r,x ) parse_make( PARSE_FOREACH,l,r,P0,s,S0,x )
-# define pif( l,r,t ) parse_make( PARSE_IF,l,r,t,S0,S0,0 )
-# define pincl( l ) parse_make( PARSE_INCLUDE,l,P0,P0,S0,S0,0 )
-# define plist( s ) parse_make( PARSE_LIST,P0,P0,P0,s,S0,0 )
-# define plocal( l,r,t ) parse_make( PARSE_LOCAL,l,r,t,S0,S0,0 )
-# define pmodule( l,r ) parse_make( PARSE_MODULE,l,r,P0,S0,S0,0 )
-# define pclass( l,r ) parse_make( PARSE_CLASS,l,r,P0,S0,S0,0 )
-# define pnull() parse_make( PARSE_NULL,P0,P0,P0,S0,S0,0 )
-# define pon( l,r ) parse_make( PARSE_ON,l,r,P0,S0,S0,0 )
-# define prule( s,p ) parse_make( PARSE_RULE,p,P0,P0,s,S0,0 )
-# define prules( l,r ) parse_make( PARSE_RULES,l,r,P0,S0,S0,0 )
-# define pset( l,r,a ) parse_make( PARSE_SET,l,r,P0,S0,S0,a )
-# define pset1( l,r,t,a ) parse_make( PARSE_SETTINGS,l,r,t,S0,S0,a )
-# define psetc( s,p,a,l ) parse_make( PARSE_SETCOMP,p,a,P0,s,S0,l )
-# define psete( s,l,s1,f ) parse_make( PARSE_SETEXEC,l,P0,P0,s,s1,f )
-# define pswitch( l,r ) parse_make( PARSE_SWITCH,l,r,P0,S0,S0,0 )
-# define pwhile( l,r ) parse_make( PARSE_WHILE,l,r,P0,S0,S0,0 )
-
-# define pnode( l,r ) parse_make( F0,l,r,P0,S0,S0,0 )
-# define psnode( s,l ) parse_make( F0,l,P0,P0,s,S0,0 )
-
-%}
-
-%%
-
-run : /* empty */
- /* do nothing */
- | rules
- { parse_save( $1.parse ); }
- ;
-
-/*
- * block - zero or more rules
- * rules - one or more rules
- * rule - any one of jam's rules
- * right-recursive so rules execute in order.
- */
-
-block : null
- { $$.parse = $1.parse; }
- | rules
- { $$.parse = $1.parse; }
- ;
-
-rules : rule
- { $$.parse = $1.parse; }
- | rule rules
- { $$.parse = prules( $1.parse, $2.parse ); }
- | `local` list assign_list_opt `;` block
- { $$.parse = plocal( $2.parse, $3.parse, $5.parse ); }
- ;
-
-null : /* empty */
- { $$.parse = pnull(); }
- ;
-
-assign_list_opt : `=` list
- { $$.parse = $2.parse; $$.number = ASSIGN_SET; }
- | null
- { $$.parse = $1.parse; $$.number = ASSIGN_APPEND; }
- ;
-
-arglist_opt : `(` lol `)`
- { $$.parse = $2.parse; }
- |
- { $$.parse = P0; }
- ;
-
-local_opt : `local`
- { $$.number = 1; }
- | /* empty */
- { $$.number = 0; }
- ;
-
-rule : `{` block `}`
- { $$.parse = $2.parse; }
- | `include` list `;`
- { $$.parse = pincl( $2.parse ); }
- | ARG lol `;`
- { $$.parse = prule( $1.string, $2.parse ); }
- | arg assign list `;`
- { $$.parse = pset( $1.parse, $3.parse, $2.number ); }
- | arg `on` list assign list `;`
- { $$.parse = pset1( $1.parse, $3.parse, $5.parse, $4.number ); }
- | `return` list `;`
- { $$.parse = $2.parse; }
- | `for` local_opt ARG `in` list `{` block `}`
- { $$.parse = pfor( $3.string, $5.parse, $7.parse, $2.number ); }
- | `switch` list `{` cases `}`
- { $$.parse = pswitch( $2.parse, $4.parse ); }
- | `if` expr `{` block `}`
- { $$.parse = pif( $2.parse, $4.parse, pnull() ); }
- | `module` list `{` block `}`
- { $$.parse = pmodule( $2.parse, $4.parse ); }
- | `class` lol `{` block `}`
- { $$.parse = pclass( $2.parse, $4.parse ); }
- | `while` expr `{` block `}`
- { $$.parse = pwhile( $2.parse, $4.parse ); }
- | `if` expr `{` block `}` `else` rule
- { $$.parse = pif( $2.parse, $4.parse, $7.parse ); }
- | local_opt `rule` ARG arglist_opt rule
- { $$.parse = psetc( $3.string, $5.parse, $4.parse, $1.number ); }
- | `on` arg rule
- { $$.parse = pon( $2.parse, $3.parse ); }
- | `actions` eflags ARG bindlist `{`
- { yymode( SCAN_STRING ); }
- STRING
- { yymode( SCAN_NORMAL ); }
- `}`
- { $$.parse = psete( $3.string,$4.parse,$7.string,$2.number ); }
- ;
-
-/*
- * assign - = or +=
- */
-
-assign : `=`
- { $$.number = ASSIGN_SET; }
- | `+=`
- { $$.number = ASSIGN_APPEND; }
- | `?=`
- { $$.number = ASSIGN_DEFAULT; }
- | `default` `=`
- { $$.number = ASSIGN_DEFAULT; }
- ;
-
-/*
- * expr - an expression for if
- */
-expr : arg
- { $$.parse = peval( EXPR_EXISTS, $1.parse, pnull() ); }
- | expr `=` expr
- { $$.parse = peval( EXPR_EQUALS, $1.parse, $3.parse ); }
- | expr `!=` expr
- { $$.parse = peval( EXPR_NOTEQ, $1.parse, $3.parse ); }
- | expr `<` expr
- { $$.parse = peval( EXPR_LESS, $1.parse, $3.parse ); }
- | expr `<=` expr
- { $$.parse = peval( EXPR_LESSEQ, $1.parse, $3.parse ); }
- | expr `>` expr
- { $$.parse = peval( EXPR_MORE, $1.parse, $3.parse ); }
- | expr `>=` expr
- { $$.parse = peval( EXPR_MOREEQ, $1.parse, $3.parse ); }
- | expr `&` expr
- { $$.parse = peval( EXPR_AND, $1.parse, $3.parse ); }
- | expr `&&` expr
- { $$.parse = peval( EXPR_AND, $1.parse, $3.parse ); }
- | expr `|` expr
- { $$.parse = peval( EXPR_OR, $1.parse, $3.parse ); }
- | expr `||` expr
- { $$.parse = peval( EXPR_OR, $1.parse, $3.parse ); }
- | arg `in` list
- { $$.parse = peval( EXPR_IN, $1.parse, $3.parse ); }
- | `!` expr
- { $$.parse = peval( EXPR_NOT, $2.parse, pnull() ); }
- | `(` expr `)`
- { $$.parse = $2.parse; }
- ;
-
-
-/*
- * cases - action elements inside a 'switch'
- * case - a single action element inside a 'switch'
- * right-recursive rule so cases can be examined in order.
- */
-
-cases : /* empty */
- { $$.parse = P0; }
- | case cases
- { $$.parse = pnode( $1.parse, $2.parse ); }
- ;
-
-case : `case` ARG `:` block
- { $$.parse = psnode( $2.string, $4.parse ); }
- ;
-
-/*
- * lol - list of lists
- * right-recursive rule so that lists can be added in order.
- */
-
-lol : list
- { $$.parse = pnode( P0, $1.parse ); }
- | list `:` lol
- { $$.parse = pnode( $3.parse, $1.parse ); }
- ;
-
-/*
- * list - zero or more args in a LIST
- * listp - list (in puncutation only mode)
- * arg - one ARG or function call
- */
-
-list : listp
- { $$.parse = $1.parse; yymode( SCAN_NORMAL ); }
- ;
-
-listp : /* empty */
- { $$.parse = pnull(); yymode( SCAN_PUNCT ); }
- | listp arg
- { $$.parse = pappend( $1.parse, $2.parse ); }
- ;
-
-arg : ARG
- { $$.parse = plist( $1.string ); }
- | `[` { yymode( SCAN_NORMAL ); } func `]`
- { $$.parse = $3.parse; }
- ;
-
-/*
- * func - a function call (inside [])
- * This needs to be split cleanly out of 'rule'
- */
-
-func : ARG lol
- { $$.parse = prule( $1.string, $2.parse ); }
- | `on` arg ARG lol
- { $$.parse = pon( $2.parse, prule( $3.string, $4.parse ) ); }
- | `on` arg `return` list
- { $$.parse = pon( $2.parse, $4.parse ); }
- ;
-
-
-/*
- * eflags - zero or more modifiers to 'executes'
- * eflag - a single modifier to 'executes'
- */
-
-eflags : /* empty */
- { $$.number = 0; }
- | eflags eflag
- { $$.number = $1.number | $2.number; }
- ;
-
-eflag : `updated`
- { $$.number = EXEC_UPDATED; }
- | `together`
- { $$.number = EXEC_TOGETHER; }
- | `ignore`
- { $$.number = EXEC_IGNORE; }
- | `quietly`
- { $$.number = EXEC_QUIETLY; }
- | `piecemeal`
- { $$.number = EXEC_PIECEMEAL; }
- | `existing`
- { $$.number = EXEC_EXISTING; }
- ;
-
-
-/*
- * bindlist - list of variable to bind for an action
- */
-
-bindlist : /* empty */
- { $$.parse = pnull(); }
- | `bind` list
- { $$.parse = $2.parse; }
- ;
-
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/jamgramtab.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/jamgramtab.h b/ext/kenlm/jam-files/engine/jamgramtab.h
deleted file mode 100644
index a0fd43f..0000000
--- a/ext/kenlm/jam-files/engine/jamgramtab.h
+++ /dev/null
@@ -1,44 +0,0 @@
- { "!", _BANG_t },
- { "!=", _BANG_EQUALS_t },
- { "&", _AMPER_t },
- { "&&", _AMPERAMPER_t },
- { "(", _LPAREN_t },
- { ")", _RPAREN_t },
- { "+=", _PLUS_EQUALS_t },
- { ":", _COLON_t },
- { ";", _SEMIC_t },
- { "<", _LANGLE_t },
- { "<=", _LANGLE_EQUALS_t },
- { "=", _EQUALS_t },
- { ">", _RANGLE_t },
- { ">=", _RANGLE_EQUALS_t },
- { "?=", _QUESTION_EQUALS_t },
- { "[", _LBRACKET_t },
- { "]", _RBRACKET_t },
- { "actions", ACTIONS_t },
- { "bind", BIND_t },
- { "case", CASE_t },
- { "class", CLASS_t },
- { "default", DEFAULT_t },
- { "else", ELSE_t },
- { "existing", EXISTING_t },
- { "for", FOR_t },
- { "if", IF_t },
- { "ignore", IGNORE_t },
- { "in", IN_t },
- { "include", INCLUDE_t },
- { "local", LOCAL_t },
- { "module", MODULE_t },
- { "on", ON_t },
- { "piecemeal", PIECEMEAL_t },
- { "quietly", QUIETLY_t },
- { "return", RETURN_t },
- { "rule", RULE_t },
- { "switch", SWITCH_t },
- { "together", TOGETHER_t },
- { "updated", UPDATED_t },
- { "while", WHILE_t },
- { "{", _LBRACE_t },
- { "|", _BAR_t },
- { "||", _BARBAR_t },
- { "}", _RBRACE_t },
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/lists.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/lists.c b/ext/kenlm/jam-files/engine/lists.c
deleted file mode 100644
index 3f2309b..0000000
--- a/ext/kenlm/jam-files/engine/lists.c
+++ /dev/null
@@ -1,475 +0,0 @@
-/*
- * Copyright 1993, 1995 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/*
- * lists.c - maintain lists of objects
- */
-
-#include "jam.h"
-#include "lists.h"
-
-#include <assert.h>
-
-static LIST * freelist[ 32 ]; /* junkpile for list_dealloc() */
-
-static unsigned get_bucket( unsigned size )
-{
- unsigned bucket = 0;
- while ( size > ( 1u << bucket ) ) ++bucket;
- return bucket;
-}
-
-static LIST * list_alloc( unsigned const size )
-{
- unsigned const bucket = get_bucket( size );
- if ( freelist[ bucket ] )
- {
- LIST * result = freelist[ bucket ];
- freelist[ bucket ] = result->impl.next;
- return result;
- }
- return (LIST *)BJAM_MALLOC( sizeof( LIST ) + ( 1u << bucket ) *
- sizeof( OBJECT * ) );
-}
-
-static void list_dealloc( LIST * l )
-{
- unsigned size = list_length( l );
- unsigned bucket;
- LIST * node = l;
-
- if ( size == 0 ) return;
-
- bucket = get_bucket( size );;
-
-#ifdef BJAM_NO_MEM_CACHE
- BJAM_FREE( node );
-#else
- node->impl.next = freelist[ bucket ];
- freelist[ bucket ] = node;
-#endif
-}
-
-/*
- * list_append() - append a list onto another one, returning total
- */
-
-LIST * list_append( LIST * l, LIST * nl )
-{
- if ( list_empty( l ) )
- return nl;
- if ( !list_empty( nl ) )
- {
- int const l_size = list_length( l );
- int const nl_size = list_length( nl );
- int const size = l_size + nl_size;
- unsigned const bucket = get_bucket( size );
-
- /* Do we need to reallocate? */
- if ( l_size <= ( 1u << ( bucket - 1 ) ) )
- {
- LIST * result = list_alloc( size );
- memcpy( list_begin( result ), list_begin( l ), l_size * sizeof(
- OBJECT * ) );
- list_dealloc( l );
- l = result;
- }
-
- l->impl.size = size;
- memcpy( list_begin( l ) + l_size, list_begin( nl ), nl_size * sizeof(
- OBJECT * ) );
- list_dealloc( nl );
- }
- return l;
-}
-
-LISTITER list_begin( LIST * l )
-{
- return l ? (LISTITER)( (char *)l + sizeof( LIST ) ) : 0;
-}
-
-LISTITER list_end( LIST * l )
-{
- return l ? list_begin( l ) + l->impl.size : 0;
-}
-
-LIST * list_new( OBJECT * value )
-{
- LIST * const head = list_alloc( 1 ) ;
- head->impl.size = 1;
- list_begin( head )[ 0 ] = value;
- return head;
-}
-
-/*
- * list_push_back() - tack a string onto the end of a list of strings
- */
-
-LIST * list_push_back( LIST * head, OBJECT * value )
-{
- unsigned int size = list_length( head );
- unsigned int i;
-
- if ( DEBUG_LISTS )
- printf( "list > %s <\n", object_str( value ) );
-
- /* If the size is a power of 2, reallocate. */
- if ( size == 0 )
- {
- head = list_alloc( 1 );
- }
- else if ( ( ( size - 1 ) & size ) == 0 )
- {
- LIST * l = list_alloc( size + 1 );
- memcpy( l, head, sizeof( LIST ) + size * sizeof( OBJECT * ) );
- list_dealloc( head );
- head = l;
- }
-
- list_begin( head )[ size ] = value;
- head->impl.size = size + 1;
-
- return head;
-}
-
-
-/*
- * list_copy() - copy a whole list of strings (nl) onto end of another (l).
- */
-
-LIST * list_copy( LIST * l )
-{
- int size = list_length( l );
- int i;
- LIST * result;
-
- if ( size == 0 ) return L0;
-
- result = list_alloc( size );
- result->impl.size = size;
- for ( i = 0; i < size; ++i )
- list_begin( result )[ i ] = object_copy( list_begin( l )[ i ] );
- return result;
-}
-
-
-LIST * list_copy_range( LIST * l, LISTITER first, LISTITER last )
-{
- if ( first == last )
- return L0;
- else
- {
- int size = last - first;
- LIST * result = list_alloc( size );
- LISTITER dest = list_begin( result );
- result->impl.size = size;
- for ( ; first != last; ++first, ++dest )
- *dest = object_copy( *first );
- return result;
- }
-}
-
-
-/*
- * list_sublist() - copy a subset of a list of strings.
- */
-
-LIST * list_sublist( LIST * l, int start, int count )
-{
- int end = start + count;
- int size = list_length( l );
- if ( start >= size ) return L0;
- if ( end > size ) end = size;
- return list_copy_range( l, list_begin( l ) + start, list_begin( l ) + end );
-}
-
-
-static int str_ptr_compare( void const * va, void const * vb )
-{
- OBJECT * a = *( (OBJECT * *)va );
- OBJECT * b = *( (OBJECT * *)vb );
- return strcmp( object_str( a ), object_str( b ) );
-}
-
-
-LIST * list_sort( LIST * l )
-{
- int len;
- int ii;
- LIST * result;
-
- if ( !l )
- return L0;
-
- len = list_length( l );
- result = list_copy( l );
-
- qsort( list_begin( result ), len, sizeof( OBJECT * ), str_ptr_compare );
-
- return result;
-}
-
-
-/*
- * list_free() - free a list of strings
- */
-
-void list_free( LIST * head )
-{
- if ( !list_empty( head ) )
- {
- LISTITER iter = list_begin( head );
- LISTITER const end = list_end( head );
- for ( ; iter != end; iter = list_next( iter ) )
- object_free( list_item( iter ) );
- list_dealloc( head );
- }
-}
-
-
-/*
- * list_pop_front() - remove the front element from a list of strings
- */
-
-LIST * list_pop_front( LIST * l )
-{
- unsigned size = list_length( l );
- assert( size );
- --size;
- object_free( list_front( l ) );
-
- if ( size == 0 )
- {
- list_dealloc( l );
- return L0;
- }
-
- if ( ( ( size - 1 ) & size ) == 0 )
- {
- LIST * const nl = list_alloc( size );
- nl->impl.size = size;
- memcpy( list_begin( nl ), list_begin( l ) + 1, size * sizeof( OBJECT * )
- );
- list_dealloc( l );
- return nl;
- }
-
- l->impl.size = size;
- memmove( list_begin( l ), list_begin( l ) + 1, size * sizeof( OBJECT * ) );
- return l;
-}
-
-LIST * list_reverse( LIST * l )
-{
- int size = list_length( l );
- if ( size == 0 ) return L0;
- {
- LIST * const result = list_alloc( size );
- int i;
- result->impl.size = size;
- for ( i = 0; i < size; ++i )
- list_begin( result )[ i ] = object_copy( list_begin( l )[ size - i -
- 1 ] );
- return result;
- }
-}
-
-int list_cmp( LIST * t, LIST * s )
-{
- int status = 0;
- LISTITER t_it = list_begin( t );
- LISTITER const t_end = list_end( t );
- LISTITER s_it = list_begin( s );
- LISTITER const s_end = list_end( s );
-
- while ( !status && ( t_it != t_end || s_it != s_end ) )
- {
- char const * st = t_it != t_end ? object_str( list_item( t_it ) ) : "";
- char const * ss = s_it != s_end ? object_str( list_item( s_it ) ) : "";
-
- status = strcmp( st, ss );
-
- t_it = t_it != t_end ? list_next( t_it ) : t_it;
- s_it = s_it != s_end ? list_next( s_it ) : s_it;
- }
-
- return status;
-}
-
-int list_is_sublist( LIST * sub, LIST * l )
-{
- LISTITER iter = list_begin( sub );
- LISTITER const end = list_end( sub );
- for ( ; iter != end; iter = list_next( iter ) )
- if ( !list_in( l, list_item( iter ) ) )
- return 0;
- return 1;
-}
-
-/*
- * list_print() - print a list of strings to stdout
- */
-
-void list_print( LIST * l )
-{
- LISTITER iter = list_begin( l ), end = list_end( l );
- if ( iter != end )
- {
- printf( "%s", object_str( list_item( iter ) ) );
- iter = list_next( iter );
- for ( ; iter != end; iter = list_next( iter ) )
- printf( " %s", object_str( list_item( iter ) ) );
- }
-}
-
-
-/*
- * list_length() - return the number of items in the list
- */
-
-int list_length( LIST * l )
-{
- return l ? l->impl.size : 0;
-}
-
-
-int list_in( LIST * l, OBJECT * value )
-{
- LISTITER iter = list_begin( l );
- LISTITER end = list_end( l );
- for ( ; iter != end; iter = list_next( iter ) )
- if ( object_equal( list_item( iter ), value ) )
- return 1;
- return 0;
-}
-
-
-LIST * list_unique( LIST * sorted_list )
-{
- LIST * result = L0;
- OBJECT * last_added = 0;
-
- LISTITER iter = list_begin( sorted_list ), end = list_end( sorted_list );
- for ( ; iter != end; iter = list_next( iter ) )
- {
- if ( !last_added || !object_equal( list_item( iter ), last_added ) )
- {
- result = list_push_back( result, object_copy( list_item( iter ) ) );
- last_added = list_item( iter );
- }
- }
- return result;
-}
-
-void list_done()
-{
- int i;
- for ( i = 0; i < sizeof( freelist ) / sizeof( freelist[ 0 ] ); ++i )
- {
- LIST * l = freelist[ i ];
- while ( l )
- {
- LIST * const tmp = l;
- l = l->impl.next;
- BJAM_FREE( tmp );
- }
- }
-}
-
-
-/*
- * lol_init() - initialize a LOL (list of lists).
- */
-
-void lol_init( LOL * lol )
-{
- lol->count = 0;
-}
-
-
-/*
- * lol_add() - append a LIST onto an LOL.
- */
-
-void lol_add( LOL * lol, LIST * l )
-{
- if ( lol->count < LOL_MAX )
- lol->list[ lol->count++ ] = l;
-}
-
-
-/*
- * lol_free() - free the LOL and its LISTs.
- */
-
-void lol_free( LOL * lol )
-{
- int i;
- for ( i = 0; i < lol->count; ++i )
- list_free( lol->list[ i ] );
- lol->count = 0;
-}
-
-
-/*
- * lol_get() - return one of the LISTs in the LOL.
- */
-
-LIST * lol_get( LOL * lol, int i )
-{
- return i < lol->count ? lol->list[ i ] : L0;
-}
-
-
-/*
- * lol_print() - debug print LISTS separated by ":".
- */
-
-void lol_print( LOL * lol )
-{
- int i;
- for ( i = 0; i < lol->count; ++i )
- {
- if ( i )
- printf( " : " );
- list_print( lol->list[ i ] );
- }
-}
-
-#ifdef HAVE_PYTHON
-
-PyObject * list_to_python( LIST * l )
-{
- PyObject * result = PyList_New( 0 );
- LISTITER iter = list_begin( l );
- LISTITER const end = list_end( l );
- for ( ; iter != end; iter = list_next( iter ) )
- {
- PyObject * s = PyString_FromString( object_str( list_item( iter ) ) );
- PyList_Append( result, s );
- Py_DECREF( s );
- }
-
- return result;
-}
-
-LIST * list_from_python( PyObject * l )
-{
- LIST * result = L0;
-
- Py_ssize_t n = PySequence_Size( l );
- Py_ssize_t i;
- for ( i = 0; i < n; ++i )
- {
- PyObject * v = PySequence_GetItem( l, i );
- result = list_push_back( result, object_new( PyString_AsString( v ) ) );
- Py_DECREF( v );
- }
-
- return result;
-}
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/lists.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/lists.h b/ext/kenlm/jam-files/engine/lists.h
deleted file mode 100644
index 3dd8fe8..0000000
--- a/ext/kenlm/jam-files/engine/lists.h
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Copyright 1993, 1995 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * lists.h - the LIST structure and routines to manipulate them
- *
- * The whole of jam relies on lists of objects as a datatype. This module, in
- * conjunction with object.c, handles these relatively efficiently.
- *
- * Structures defined:
- *
- * LIST - list of OBJECTs
- * LOL - list of LISTs
- *
- * External routines:
- *
- * list_append() - append a list onto another one, returning total
- * list_new() - tack an object onto the end of a list of objects
- * list_copy() - copy a whole list of objects
- * list_sublist() - copy a subset of a list of objects
- * list_free() - free a list of objects
- * list_print() - print a list of objects to stdout
- * list_length() - return the number of items in the list
- *
- * lol_init() - initialize a LOL (list of lists)
- * lol_add() - append a LIST onto an LOL
- * lol_free() - free the LOL and its LISTs
- * lol_get() - return one of the LISTs in the LOL
- * lol_print() - debug print LISTS separated by ":"
- */
-
-#ifndef LISTS_DWA20011022_H
-#define LISTS_DWA20011022_H
-
-#include "object.h"
-
-#ifdef HAVE_PYTHON
-# include <Python.h>
-#endif
-
-/*
- * LIST - list of strings
- */
-
-typedef struct _list {
- union {
- int size;
- struct _list * next;
- OBJECT * align;
- } impl;
-} LIST;
-
-typedef OBJECT * * LISTITER;
-
-/*
- * LOL - list of LISTs
- */
-
-#define LOL_MAX 19
-typedef struct _lol {
- int count;
- LIST * list[ LOL_MAX ];
-} LOL;
-
-LIST * list_new( OBJECT * value );
-LIST * list_append( LIST * destination, LIST * source );
-LIST * list_copy( LIST * );
-LIST * list_copy_range( LIST * destination, LISTITER first, LISTITER last );
-void list_free( LIST * head );
-LIST * list_push_back( LIST * head, OBJECT * value );
-void list_print( LIST * );
-int list_length( LIST * );
-LIST * list_sublist( LIST *, int start, int count );
-LIST * list_pop_front( LIST * );
-LIST * list_sort( LIST * );
-LIST * list_unique( LIST * sorted_list );
-int list_in( LIST *, OBJECT * value );
-LIST * list_reverse( LIST * );
-int list_cmp( LIST * lhs, LIST * rhs );
-int list_is_sublist( LIST * sub, LIST * l );
-void list_done();
-
-LISTITER list_begin( LIST * );
-LISTITER list_end( LIST * );
-#define list_next( it ) ((it) + 1)
-#define list_item( it ) (*(it))
-#define list_empty( l ) ((l) == L0)
-#define list_front( l ) list_item( list_begin( l ) )
-
-#define L0 ((LIST *)0)
-
-void lol_add( LOL *, LIST * );
-void lol_init( LOL * );
-void lol_free( LOL * );
-LIST * lol_get( LOL *, int i );
-void lol_print( LOL * );
-void lol_build( LOL *, char const * * elements );
-
-#ifdef HAVE_PYTHON
-PyObject * list_to_python( LIST * );
-LIST * list_from_python( PyObject * );
-#endif
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/make.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/make.c b/ext/kenlm/jam-files/engine/make.c
deleted file mode 100644
index afc8bb9..0000000
--- a/ext/kenlm/jam-files/engine/make.c
+++ /dev/null
@@ -1,908 +0,0 @@
-/*
- * Copyright 1993, 1995 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * make.c - bring a target up to date, once rules are in place.
- *
- * This modules controls the execution of rules to bring a target and its
- * dependencies up to date. It is invoked after the targets, rules, et. al.
- * described in rules.h are created by the interpreting jam files.
- *
- * This file contains the main make() entry point and the first pass make0().
- * The second pass, make1(), which actually does the command execution, is in
- * make1.c.
- *
- * External routines:
- * make() - make a target, given its name
- *
- * Internal routines:
- * make0() - bind and scan everything to make a TARGET
- * make0sort() - reorder TARGETS chain by their time (newest to oldest)
- */
-
-#include "jam.h"
-#include "make.h"
-
-#include "command.h"
-#ifdef OPT_HEADER_CACHE_EXT
-# include "hcache.h"
-#endif
-#include "headers.h"
-#include "lists.h"
-#include "object.h"
-#include "parse.h"
-#include "rules.h"
-#include "search.h"
-#include "timestamp.h"
-#include "variable.h"
-
-#include <assert.h>
-
-#ifndef max
-# define max(a,b) ((a)>(b)?(a):(b))
-#endif
-
-static TARGETS * make0sort( TARGETS * c );
-
-#ifdef OPT_GRAPH_DEBUG_EXT
- static void dependGraphOutput( TARGET * t, int depth );
-#endif
-
-static char const * target_fate[] =
-{
- "init", /* T_FATE_INIT */
- "making", /* T_FATE_MAKING */
- "stable", /* T_FATE_STABLE */
- "newer", /* T_FATE_NEWER */
- "temp", /* T_FATE_ISTMP */
- "touched", /* T_FATE_TOUCHED */
- "rebuild", /* T_FATE_REBUILD */
- "missing", /* T_FATE_MISSING */
- "needtmp", /* T_FATE_NEEDTMP */
- "old", /* T_FATE_OUTDATED */
- "update", /* T_FATE_UPDATE */
- "nofind", /* T_FATE_CANTFIND */
- "nomake" /* T_FATE_CANTMAKE */
-};
-
-static char const * target_bind[] =
-{
- "unbound",
- "missing",
- "parents",
- "exists",
-};
-
-#define spaces(x) ( " " + ( x > 20 ? 0 : 20-x ) )
-
-
-/*
- * make() - make a target, given its name.
- */
-
-int make( LIST * targets, int anyhow )
-{
- COUNTS counts[ 1 ];
- int status = 0; /* 1 if anything fails */
-
-#ifdef OPT_HEADER_CACHE_EXT
- hcache_init();
-#endif
-
- memset( (char *)counts, 0, sizeof( *counts ) );
-
- /* First bind all targets with LOCATE_TARGET setting. This is needed to
- * correctly handle dependencies to generated headers.
- */
- bind_explicitly_located_targets();
-
- {
- LISTITER iter, end;
- PROFILE_ENTER( MAKE_MAKE0 );
- for ( iter = list_begin( targets ), end = list_end( targets ); iter != end; iter = list_next( iter ) )
- {
- TARGET * t = bindtarget( list_item( iter ) );
- if ( t->fate == T_FATE_INIT )
- make0( t, 0, 0, counts, anyhow, 0 );
- }
- PROFILE_EXIT( MAKE_MAKE0 );
- }
-
-#ifdef OPT_GRAPH_DEBUG_EXT
- if ( DEBUG_GRAPH )
- {
- LISTITER iter, end;
- for ( iter = list_begin( targets ), end = list_end( targets ); iter != end; iter = list_next( iter ) )
- dependGraphOutput( bindtarget( list_item( iter ) ), 0 );
- }
-#endif
-
- if ( DEBUG_MAKE )
- {
- if ( counts->targets )
- printf( "...found %d target%s...\n", counts->targets,
- counts->targets > 1 ? "s" : "" );
- if ( counts->temp )
- printf( "...using %d temp target%s...\n", counts->temp,
- counts->temp > 1 ? "s" : "" );
- if ( counts->updating )
- printf( "...updating %d target%s...\n", counts->updating,
- counts->updating > 1 ? "s" : "" );
- if ( counts->cantfind )
- printf( "...can't find %d target%s...\n", counts->cantfind,
- counts->cantfind > 1 ? "s" : "" );
- if ( counts->cantmake )
- printf( "...can't make %d target%s...\n", counts->cantmake,
- counts->cantmake > 1 ? "s" : "" );
- }
-
- status = counts->cantfind || counts->cantmake;
-
- {
- PROFILE_ENTER( MAKE_MAKE1 );
- status |= make1( targets );
- PROFILE_EXIT( MAKE_MAKE1 );
- }
-
- return status;
-}
-
-
-/* Force any dependants of t that have already at least begun being visited by
- * make0() to be updated.
- */
-
-static void update_dependants( TARGET * t )
-{
- TARGETS * q;
-
- for ( q = t->dependants; q; q = q->next )
- {
- TARGET * p = q->target;
- char fate0 = p->fate;
-
- /* If we have already at least begun visiting it and we are not already
- * rebuilding it for other reasons.
- */
- if ( ( fate0 != T_FATE_INIT ) && ( fate0 < T_FATE_BUILD ) )
- {
- p->fate = T_FATE_UPDATE;
-
- if ( DEBUG_FATE )
- {
- printf( "fate change %s from %s to %s (as dependant of %s)\n",
- object_str( p->name ), target_fate[ (int) fate0 ], target_fate[ (int) p->fate ], object_str( t->name ) );
- }
-
- /* If we are done visiting it, go back and make sure its dependants
- * get rebuilt.
- */
- if ( fate0 > T_FATE_MAKING )
- update_dependants( p );
- }
- }
-}
-
-
-/*
- * Make sure that all of t's rebuilds get rebuilt.
- */
-
-static void force_rebuilds( TARGET * t )
-{
- TARGETS * d;
- for ( d = t->rebuilds; d; d = d->next )
- {
- TARGET * r = d->target;
-
- /* If it is not already being rebuilt for other reasons. */
- if ( r->fate < T_FATE_BUILD )
- {
- if ( DEBUG_FATE )
- printf( "fate change %s from %s to %s (by rebuild)\n",
- object_str( r->name ), target_fate[ (int) r->fate ], target_fate[ T_FATE_REBUILD ] );
-
- /* Force rebuild it. */
- r->fate = T_FATE_REBUILD;
-
- /* And make sure its dependants are updated too. */
- update_dependants( r );
- }
- }
-}
-
-
-int make0rescan( TARGET * t, TARGET * rescanning )
-{
- int result = 0;
- TARGETS * c;
-
- /* Check whether we have already found a cycle. */
- if ( target_scc( t ) == rescanning )
- return 1;
-
- /* If we have already visited this node, ignore it. */
- if ( t->rescanning == rescanning )
- return 0;
-
- /* If t is already updated, ignore it. */
- if ( t->scc_root == NULL && t->progress > T_MAKE_ACTIVE )
- return 0;
-
- t->rescanning = rescanning;
- for ( c = t->depends; c; c = c->next )
- {
- TARGET * dependency = c->target;
- /* Always start at the root of each new strongly connected component. */
- if ( target_scc( dependency ) != target_scc( t ) )
- dependency = target_scc( dependency );
- result |= make0rescan( dependency, rescanning );
-
- /* Make sure that we pick up the new include node. */
- if ( c->target->includes == rescanning )
- result = 1;
- }
- if ( result && t->scc_root == NULL )
- {
- t->scc_root = rescanning;
- rescanning->depends = targetentry( rescanning->depends, t );
- }
- return result;
-}
-
-
-/*
- * make0() - bind and scan everything to make a TARGET.
- *
- * Recursively binds a target, searches for #included headers, calls itself on
- * those headers and any dependencies.
- */
-
-void make0
-(
- TARGET * t,
- TARGET * p, /* parent */
- int depth, /* for display purposes */
- COUNTS * counts, /* for reporting */
- int anyhow,
- TARGET * rescanning
-) /* forcibly touch all (real) targets */
-{
- TARGETS * c;
- TARGET * ptime = t;
- TARGET * located_target = 0;
- timestamp last;
- timestamp leaf;
- timestamp hlast;
- int fate;
- char const * flag = "";
- SETTINGS * s;
-
-#ifdef OPT_GRAPH_DEBUG_EXT
- int savedFate;
- int oldTimeStamp;
-#endif
-
- if ( DEBUG_MAKEPROG )
- printf( "make\t--\t%s%s\n", spaces( depth ), object_str( t->name ) );
-
- /*
- * Step 1: Initialize.
- */
-
- if ( DEBUG_MAKEPROG )
- printf( "make\t--\t%s%s\n", spaces( depth ), object_str( t->name ) );
-
- t->fate = T_FATE_MAKING;
- t->depth = depth;
-
- /*
- * Step 2: Under the influence of "on target" variables, bind the target and
- * search for headers.
- */
-
- /* Step 2a: Set "on target" variables. */
- s = copysettings( t->settings );
- pushsettings( root_module(), s );
-
- /* Step 2b: Find and timestamp the target file (if it is a file). */
- if ( ( t->binding == T_BIND_UNBOUND ) && !( t->flags & T_FLAG_NOTFILE ) )
- {
- OBJECT * another_target;
- object_free( t->boundname );
- t->boundname = search( t->name, &t->time, &another_target,
- t->flags & T_FLAG_ISFILE );
- /* If it was detected that this target refers to an already existing and
- * bound target, we add a dependency so that every target depending on
- * us will depend on that other target as well.
- */
- if ( another_target )
- located_target = bindtarget( another_target );
-
- t->binding = timestamp_empty( &t->time )
- ? T_BIND_MISSING
- : T_BIND_EXISTS;
- }
-
- /* INTERNAL, NOTFILE header nodes have the time of their parents. */
- if ( p && ( t->flags & T_FLAG_INTERNAL ) )
- ptime = p;
-
- /* If temp file does not exist but parent does, use parent. */
- if ( p && ( t->flags & T_FLAG_TEMP ) &&
- ( t->binding == T_BIND_MISSING ) &&
- ( p->binding != T_BIND_MISSING ) )
- {
- t->binding = T_BIND_PARENTS;
- ptime = p;
- }
-
-#ifdef OPT_SEMAPHORE
- {
- LIST * var = var_get( root_module(), constant_JAM_SEMAPHORE );
- if ( !list_empty( var ) )
- {
- TARGET * const semaphore = bindtarget( list_front( var ) );
- semaphore->progress = T_MAKE_SEMAPHORE;
- t->semaphore = semaphore;
- }
- }
-#endif
-
- /* Step 2c: If its a file, search for headers. */
- if ( t->binding == T_BIND_EXISTS )
- headers( t );
-
- /* Step 2d: reset "on target" variables. */
- popsettings( root_module(), s );
- freesettings( s );
-
- /*
- * Pause for a little progress reporting.
- */
-
- if ( DEBUG_BIND )
- {
- if ( !object_equal( t->name, t->boundname ) )
- printf( "bind\t--\t%s%s: %s\n", spaces( depth ),
- object_str( t->name ), object_str( t->boundname ) );
-
- switch ( t->binding )
- {
- case T_BIND_UNBOUND:
- case T_BIND_MISSING:
- case T_BIND_PARENTS:
- printf( "time\t--\t%s%s: %s\n", spaces( depth ),
- object_str( t->name ), target_bind[ (int)t->binding ] );
- break;
-
- case T_BIND_EXISTS:
- printf( "time\t--\t%s%s: %s\n", spaces( depth ),
- object_str( t->name ), timestamp_str( &t->time ) );
- break;
- }
- }
-
- /*
- * Step 3: Recursively make0() dependencies & headers.
- */
-
- /* Step 3a: Recursively make0() dependencies. */
- for ( c = t->depends; c; c = c->next )
- {
- int const internal = t->flags & T_FLAG_INTERNAL;
-
- /* Warn about circular deps, except for includes, which include each
- * other alot.
- */
- if ( c->target->fate == T_FATE_INIT )
- make0( c->target, ptime, depth + 1, counts, anyhow, rescanning );
- else if ( c->target->fate == T_FATE_MAKING && !internal )
- printf( "warning: %s depends on itself\n", object_str(
- c->target->name ) );
- else if ( c->target->fate != T_FATE_MAKING && rescanning )
- make0rescan( c->target, rescanning );
- if ( rescanning && c->target->includes && c->target->includes->fate !=
- T_FATE_MAKING )
- make0rescan( target_scc( c->target->includes ), rescanning );
- }
-
- if ( located_target )
- {
- if ( located_target->fate == T_FATE_INIT )
- make0( located_target, ptime, depth + 1, counts, anyhow, rescanning
- );
- else if ( located_target->fate != T_FATE_MAKING && rescanning )
- make0rescan( located_target, rescanning );
- }
-
- /* Step 3b: Recursively make0() internal includes node. */
- if ( t->includes )
- make0( t->includes, p, depth + 1, counts, anyhow, rescanning );
-
- /* Step 3c: Add dependencies' includes to our direct dependencies. */
- {
- TARGETS * incs = 0;
- for ( c = t->depends; c; c = c->next )
- if ( c->target->includes )
- incs = targetentry( incs, c->target->includes );
- t->depends = targetchain( t->depends, incs );
- }
-
- if ( located_target )
- t->depends = targetentry( t->depends, located_target );
-
- /* Step 3d: Detect cycles. */
- {
- int cycle_depth = depth;
- for ( c = t->depends; c; c = c->next )
- {
- TARGET * scc_root = target_scc( c->target );
- if ( scc_root->fate == T_FATE_MAKING &&
- ( !scc_root->includes ||
- scc_root->includes->fate != T_FATE_MAKING ) )
- {
- if ( scc_root->depth < cycle_depth )
- {
- cycle_depth = scc_root->depth;
- t->scc_root = scc_root;
- }
- }
- }
- }
-
- /*
- * Step 4: Compute time & fate.
- */
-
- /* Step 4a: Pick up dependencies' time and fate. */
- timestamp_clear( &last );
- timestamp_clear( &leaf );
- fate = T_FATE_STABLE;
- for ( c = t->depends; c; c = c->next )
- {
- /* If we are in a different strongly connected component, pull
- * timestamps from the root.
- */
- if ( c->target->scc_root )
- {
- TARGET * const scc_root = target_scc( c->target );
- if ( scc_root != t->scc_root )
- {
- timestamp_max( &c->target->leaf, &c->target->leaf,
- &scc_root->leaf );
- timestamp_max( &c->target->time, &c->target->time,
- &scc_root->time );
- c->target->fate = max( c->target->fate, scc_root->fate );
- }
- }
-
- /* If LEAVES has been applied, we only heed the timestamps of the leaf
- * source nodes.
- */
- timestamp_max( &leaf, &leaf, &c->target->leaf );
- if ( t->flags & T_FLAG_LEAVES )
- {
- timestamp_copy( &last, &leaf );
- continue;
- }
- timestamp_max( &last, &last, &c->target->time );
- fate = max( fate, c->target->fate );
-
-#ifdef OPT_GRAPH_DEBUG_EXT
- if ( DEBUG_FATE )
- if ( fate < c->target->fate )
- printf( "fate change %s from %s to %s by dependency %s\n",
- object_str( t->name ), target_fate[ (int)fate ],
- target_fate[ (int)c->target->fate ], object_str(
- c->target->name ) );
-#endif
- }
-
- /* Step 4b: Pick up included headers time. */
-
- /*
- * If a header is newer than a temp source that includes it, the temp source
- * will need building.
- */
- if ( t->includes )
- timestamp_copy( &hlast, &t->includes->time );
- else
- timestamp_clear( &hlast );
-
- /* Step 4c: handle NOUPDATE oddity.
- *
- * If a NOUPDATE file exists, mark it as having eternally old dependencies.
- * Do not inherit our fate from our dependencies. Decide fate based only on
- * other flags and our binding (done later).
- */
- if ( t->flags & T_FLAG_NOUPDATE )
- {
-#ifdef OPT_GRAPH_DEBUG_EXT
- if ( DEBUG_FATE )
- if ( fate != T_FATE_STABLE )
- printf( "fate change %s back to stable, NOUPDATE.\n",
- object_str( t->name ) );
-#endif
-
- timestamp_clear( &last );
- timestamp_clear( &t->time );
-
- /* Do not inherit our fate from our dependencies. Decide fate based only
- * upon other flags and our binding (done later).
- */
- fate = T_FATE_STABLE;
- }
-
- /* Step 4d: Determine fate: rebuild target or what? */
-
- /*
- In English:
- If can not find or make child, can not make target.
- If children changed, make target.
- If target missing, make it.
- If children newer, make target.
- If temp's children newer than parent, make temp.
- If temp's headers newer than parent, make temp.
- If deliberately touched, make it.
- If up-to-date temp file present, use it.
- If target newer than non-notfile parent, mark target newer.
- Otherwise, stable!
-
- Note this block runs from least to most stable: as we make it further
- down the list, the target's fate gets more stable.
- */
-
-#ifdef OPT_GRAPH_DEBUG_EXT
- savedFate = fate;
- oldTimeStamp = 0;
-#endif
-
- if ( fate >= T_FATE_BROKEN )
- {
- fate = T_FATE_CANTMAKE;
- }
- else if ( fate >= T_FATE_SPOIL )
- {
- fate = T_FATE_UPDATE;
- }
- else if ( t->binding == T_BIND_MISSING )
- {
- fate = T_FATE_MISSING;
- }
- else if ( t->binding == T_BIND_EXISTS && timestamp_cmp( &last, &t->time ) >
- 0 )
- {
-#ifdef OPT_GRAPH_DEBUG_EXT
- oldTimeStamp = 1;
-#endif
- fate = T_FATE_OUTDATED;
- }
- else if ( t->binding == T_BIND_PARENTS && timestamp_cmp( &last, &p->time ) >
- 0 )
- {
-#ifdef OPT_GRAPH_DEBUG_EXT
- oldTimeStamp = 1;
-#endif
- fate = T_FATE_NEEDTMP;
- }
- else if ( t->binding == T_BIND_PARENTS && timestamp_cmp( &hlast, &p->time )
- > 0 )
- {
- fate = T_FATE_NEEDTMP;
- }
- else if ( t->flags & T_FLAG_TOUCHED )
- {
- fate = T_FATE_TOUCHED;
- }
- else if ( anyhow && !( t->flags & T_FLAG_NOUPDATE ) )
- {
- fate = T_FATE_TOUCHED;
- }
- else if ( t->binding == T_BIND_EXISTS && ( t->flags & T_FLAG_TEMP ) )
- {
- fate = T_FATE_ISTMP;
- }
- else if ( t->binding == T_BIND_EXISTS && p && p->binding != T_BIND_UNBOUND
- && timestamp_cmp( &t->time, &p->time ) > 0 )
- {
-#ifdef OPT_GRAPH_DEBUG_EXT
- oldTimeStamp = 1;
-#endif
- fate = T_FATE_NEWER;
- }
- else
- {
- fate = T_FATE_STABLE;
- }
-#ifdef OPT_GRAPH_DEBUG_EXT
- if ( DEBUG_FATE && ( fate != savedFate ) )
- {
- if ( savedFate == T_FATE_STABLE )
- printf( "fate change %s set to %s%s\n", object_str( t->name ),
- target_fate[ fate ], oldTimeStamp ? " (by timestamp)" : "" );
- else
- printf( "fate change %s from %s to %s%s\n", object_str( t->name ),
- target_fate[ savedFate ], target_fate[ fate ], oldTimeStamp ?
- " (by timestamp)" : "" );
- }
-#endif
-
- /* Step 4e: Handle missing files. */
- /* If it is missing and there are no actions to create it, boom. */
- /* If we can not make a target we do not care about it, okay. */
- /* We could insist that there are updating actions for all missing */
- /* files, but if they have dependencies we just pretend it is a NOTFILE. */
-
- if ( ( fate == T_FATE_MISSING ) && !t->actions && !t->depends )
- {
- if ( t->flags & T_FLAG_NOCARE )
- {
-#ifdef OPT_GRAPH_DEBUG_EXT
- if ( DEBUG_FATE )
- printf( "fate change %s to STABLE from %s, "
- "no actions, no dependencies and do not care\n",
- object_str( t->name ), target_fate[ fate ] );
-#endif
- fate = T_FATE_STABLE;
- }
- else
- {
- printf( "don't know how to make %s\n", object_str( t->name ) );
- fate = T_FATE_CANTFIND;
- }
- }
-
- /* Step 4f: Propagate dependencies' time & fate. */
- /* Set leaf time to be our time only if this is a leaf. */
-
- timestamp_max( &t->time, &t->time, &last );
- timestamp_copy( &t->leaf, timestamp_empty( &leaf ) ? &t->time : &leaf );
- /* This target's fate may have been updated by virtue of following some
- * target's rebuilds list, so only allow it to be increased to the fate we
- * have calculated. Otherwise, grab its new fate.
- */
- if ( fate > t->fate )
- t->fate = fate;
- else
- fate = t->fate;
-
- /* Step 4g: If this target needs to be built, force rebuild everything in
- * its rebuilds list.
- */
- if ( ( fate >= T_FATE_BUILD ) && ( fate < T_FATE_BROKEN ) )
- force_rebuilds( t );
-
- /*
- * Step 5: Sort dependencies by their update time.
- */
-
- if ( globs.newestfirst )
- t->depends = make0sort( t->depends );
-
- /*
- * Step 6: A little harmless tabulating for tracing purposes.
- */
-
- /* Do not count or report interal includes nodes. */
- if ( t->flags & T_FLAG_INTERNAL )
- return;
-
- if ( counts )
- {
-#ifdef OPT_IMPROVED_PATIENCE_EXT
- ++counts->targets;
-#else
- if ( !( ++counts->targets % 1000 ) && DEBUG_MAKE )
- {
- printf( "...patience...\n" );
- fflush(stdout);
- }
-#endif
-
- if ( fate == T_FATE_ISTMP )
- ++counts->temp;
- else if ( fate == T_FATE_CANTFIND )
- ++counts->cantfind;
- else if ( ( fate == T_FATE_CANTMAKE ) && t->actions )
- ++counts->cantmake;
- else if ( ( fate >= T_FATE_BUILD ) && ( fate < T_FATE_BROKEN ) &&
- t->actions )
- ++counts->updating;
- }
-
- if ( !( t->flags & T_FLAG_NOTFILE ) && ( fate >= T_FATE_SPOIL ) )
- flag = "+";
- else if ( t->binding == T_BIND_EXISTS && p && timestamp_cmp( &t->time,
- &p->time ) > 0 )
- flag = "*";
-
- if ( DEBUG_MAKEPROG )
- printf( "made%s\t%s\t%s%s\n", flag, target_fate[ (int)t->fate ],
- spaces( depth ), object_str( t->name ) );
-}
-
-
-#ifdef OPT_GRAPH_DEBUG_EXT
-
-static char const * target_name( TARGET * t )
-{
- static char buf[ 1000 ];
- if ( t->flags & T_FLAG_INTERNAL )
- {
- sprintf( buf, "%s (internal node)", object_str( t->name ) );
- return buf;
- }
- return object_str( t->name );
-}
-
-
-/*
- * dependGraphOutput() - output the DG after make0 has run.
- */
-
-static void dependGraphOutput( TARGET * t, int depth )
-{
- TARGETS * c;
-
- if ( ( t->flags & T_FLAG_VISITED ) || !t->name || !t->boundname )
- return;
-
- t->flags |= T_FLAG_VISITED;
-
- switch ( t->fate )
- {
- case T_FATE_TOUCHED:
- case T_FATE_MISSING:
- case T_FATE_OUTDATED:
- case T_FATE_UPDATE:
- printf( "->%s%2d Name: %s\n", spaces( depth ), depth, target_name( t
- ) );
- break;
- default:
- printf( " %s%2d Name: %s\n", spaces( depth ), depth, target_name( t
- ) );
- break;
- }
-
- if ( !object_equal( t->name, t->boundname ) )
- printf( " %s Loc: %s\n", spaces( depth ), object_str( t->boundname )
- );
-
- switch ( t->fate )
- {
- case T_FATE_STABLE:
- printf( " %s : Stable\n", spaces( depth ) );
- break;
- case T_FATE_NEWER:
- printf( " %s : Newer\n", spaces( depth ) );
- break;
- case T_FATE_ISTMP:
- printf( " %s : Up to date temp file\n", spaces( depth ) );
- break;
- case T_FATE_NEEDTMP:
- printf( " %s : Temporary file, to be updated\n", spaces( depth )
- );
- break;
- case T_FATE_TOUCHED:
- printf( " %s : Been touched, updating it\n", spaces( depth ) );
- break;
- case T_FATE_MISSING:
- printf( " %s : Missing, creating it\n", spaces( depth ) );
- break;
- case T_FATE_OUTDATED:
- printf( " %s : Outdated, updating it\n", spaces( depth ) );
- break;
- case T_FATE_REBUILD:
- printf( " %s : Rebuild, updating it\n", spaces( depth ) );
- break;
- case T_FATE_UPDATE:
- printf( " %s : Updating it\n", spaces( depth ) );
- break;
- case T_FATE_CANTFIND:
- printf( " %s : Can not find it\n", spaces( depth ) );
- break;
- case T_FATE_CANTMAKE:
- printf( " %s : Can make it\n", spaces( depth ) );
- break;
- }
-
- if ( t->flags & ~T_FLAG_VISITED )
- {
- printf( " %s : ", spaces( depth ) );
- if ( t->flags & T_FLAG_TEMP ) printf( "TEMPORARY " );
- if ( t->flags & T_FLAG_NOCARE ) printf( "NOCARE " );
- if ( t->flags & T_FLAG_NOTFILE ) printf( "NOTFILE " );
- if ( t->flags & T_FLAG_TOUCHED ) printf( "TOUCHED " );
- if ( t->flags & T_FLAG_LEAVES ) printf( "LEAVES " );
- if ( t->flags & T_FLAG_NOUPDATE ) printf( "NOUPDATE " );
- printf( "\n" );
- }
-
- for ( c = t->depends; c; c = c->next )
- {
- printf( " %s : Depends on %s (%s)", spaces( depth ),
- target_name( c->target ), target_fate[ (int)c->target->fate ] );
- if ( !timestamp_cmp( &c->target->time, &t->time ) )
- printf( " (max time)");
- printf( "\n" );
- }
-
- for ( c = t->depends; c; c = c->next )
- dependGraphOutput( c->target, depth + 1 );
-}
-#endif
-
-
-/*
- * make0sort() - reorder TARGETS chain by their time (newest to oldest).
- *
- * We walk chain, taking each item and inserting it on the sorted result, with
- * newest items at the front. This involves updating each of the TARGETS'
- * c->next and c->tail. Note that we make c->tail a valid prev pointer for every
- * entry. Normally, it is only valid at the head, where prev == tail. Note also
- * that while tail is a loop, next ends at the end of the chain.
- */
-
-static TARGETS * make0sort( TARGETS * chain )
-{
- PROFILE_ENTER( MAKE_MAKE0SORT );
-
- TARGETS * result = 0;
-
- /* Walk the current target list. */
- while ( chain )
- {
- TARGETS * c = chain;
- TARGETS * s = result;
-
- chain = chain->next;
-
- /* Find point s in result for c. */
- while ( s && timestamp_cmp( &s->target->time, &c->target->time ) > 0 )
- s = s->next;
-
- /* Insert c in front of s (might be 0). */
- c->next = s; /* good even if s = 0 */
- if ( result == s ) result = c; /* new head of chain? */
- if ( !s ) s = result; /* wrap to ensure a next */
- if ( result != c ) s->tail->next = c; /* not head? be prev's next */
- c->tail = s->tail; /* take on next's prev */
- s->tail = c; /* make next's prev us */
- }
-
- PROFILE_EXIT( MAKE_MAKE0SORT );
- return result;
-}
-
-
-static LIST * targets_to_update_ = L0;
-
-
-void mark_target_for_updating( OBJECT * target )
-{
- targets_to_update_ = list_push_back( targets_to_update_, object_copy(
- target ) );
-}
-
-
-LIST * targets_to_update()
-{
- return targets_to_update_;
-}
-
-
-void clear_targets_to_update()
-{
- list_free( targets_to_update_ );
- targets_to_update_ = L0;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/make.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/make.h b/ext/kenlm/jam-files/engine/make.h
deleted file mode 100644
index 2c3ba16..0000000
--- a/ext/kenlm/jam-files/engine/make.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright 1993, 1995 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/*
- * make.h - bring a target up to date, once rules are in place
- */
-
-#ifndef MAKE_SW20111118_H
-#define MAKE_SW20111118_H
-
-#include "lists.h"
-#include "object.h"
-#include "rules.h"
-
-int make( LIST * targets, int anyhow );
-int make1( LIST * t );
-
-typedef struct {
- int temp;
- int updating;
- int cantfind;
- int cantmake;
- int targets;
- int made;
-} COUNTS ;
-
-
-void make0( TARGET * t, TARGET * p, int depth, COUNTS * counts, int anyhow,
- TARGET * rescanning );
-
-
-/* Specifies that the target should be updated. */
-void mark_target_for_updating( OBJECT * target );
-
-/* Returns targets previously passed to mark_target_for_updating(). */
-LIST * targets_to_update();
-
-/* Clears/unmarks all targets currently marked for update. */
-void clear_targets_to_update();
-
-#endif
[03/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/double-conversion/fast-dtoa.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/double-conversion/fast-dtoa.cc b/ext/kenlm/util/double-conversion/fast-dtoa.cc
deleted file mode 100644
index 1a0f823..0000000
--- a/ext/kenlm/util/double-conversion/fast-dtoa.cc
+++ /dev/null
@@ -1,664 +0,0 @@
-// Copyright 2012 the V8 project authors. All rights reserved.
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following
-// disclaimer in the documentation and/or other materials provided
-// with the distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived
-// from this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#include "fast-dtoa.h"
-
-#include "cached-powers.h"
-#include "diy-fp.h"
-#include "ieee.h"
-
-namespace double_conversion {
-
-// The minimal and maximal target exponent define the range of w's binary
-// exponent, where 'w' is the result of multiplying the input by a cached power
-// of ten.
-//
-// A different range might be chosen on a different platform, to optimize digit
-// generation, but a smaller range requires more powers of ten to be cached.
-static const int kMinimalTargetExponent = -60;
-static const int kMaximalTargetExponent = -32;
-
-
-// Adjusts the last digit of the generated number, and screens out generated
-// solutions that may be inaccurate. A solution may be inaccurate if it is
-// outside the safe interval, or if we cannot prove that it is closer to the
-// input than a neighboring representation of the same length.
-//
-// Input: * buffer containing the digits of too_high / 10^kappa
-// * the buffer's length
-// * distance_too_high_w == (too_high - w).f() * unit
-// * unsafe_interval == (too_high - too_low).f() * unit
-// * rest = (too_high - buffer * 10^kappa).f() * unit
-// * ten_kappa = 10^kappa * unit
-// * unit = the common multiplier
-// Output: returns true if the buffer is guaranteed to contain the closest
-// representable number to the input.
-// Modifies the generated digits in the buffer to approach (round towards) w.
-static bool RoundWeed(Vector<char> buffer,
- int length,
- uint64_t distance_too_high_w,
- uint64_t unsafe_interval,
- uint64_t rest,
- uint64_t ten_kappa,
- uint64_t unit) {
- uint64_t small_distance = distance_too_high_w - unit;
- uint64_t big_distance = distance_too_high_w + unit;
- // Let w_low = too_high - big_distance, and
- // w_high = too_high - small_distance.
- // Note: w_low < w < w_high
- //
- // The real w (* unit) must lie somewhere inside the interval
- // ]w_low; w_high[ (often written as "(w_low; w_high)")
-
- // Basically the buffer currently contains a number in the unsafe interval
- // ]too_low; too_high[ with too_low < w < too_high
- //
- // too_high - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- // ^v 1 unit ^ ^ ^ ^
- // boundary_high --------------------- . . . .
- // ^v 1 unit . . . .
- // - - - - - - - - - - - - - - - - - - - + - - + - - - - - - . .
- // . . ^ . .
- // . big_distance . . .
- // . . . . rest
- // small_distance . . . .
- // v . . . .
- // w_high - - - - - - - - - - - - - - - - - - . . . .
- // ^v 1 unit . . . .
- // w ---------------------------------------- . . . .
- // ^v 1 unit v . . .
- // w_low - - - - - - - - - - - - - - - - - - - - - . . .
- // . . v
- // buffer --------------------------------------------------+-------+--------
- // . .
- // safe_interval .
- // v .
- // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - .
- // ^v 1 unit .
- // boundary_low ------------------------- unsafe_interval
- // ^v 1 unit v
- // too_low - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- //
- //
- // Note that the value of buffer could lie anywhere inside the range too_low
- // to too_high.
- //
- // boundary_low, boundary_high and w are approximations of the real boundaries
- // and v (the input number). They are guaranteed to be precise up to one unit.
- // In fact the error is guaranteed to be strictly less than one unit.
- //
- // Anything that lies outside the unsafe interval is guaranteed not to round
- // to v when read again.
- // Anything that lies inside the safe interval is guaranteed to round to v
- // when read again.
- // If the number inside the buffer lies inside the unsafe interval but not
- // inside the safe interval then we simply do not know and bail out (returning
- // false).
- //
- // Similarly we have to take into account the imprecision of 'w' when finding
- // the closest representation of 'w'. If we have two potential
- // representations, and one is closer to both w_low and w_high, then we know
- // it is closer to the actual value v.
- //
- // By generating the digits of too_high we got the largest (closest to
- // too_high) buffer that is still in the unsafe interval. In the case where
- // w_high < buffer < too_high we try to decrement the buffer.
- // This way the buffer approaches (rounds towards) w.
- // There are 3 conditions that stop the decrementation process:
- // 1) the buffer is already below w_high
- // 2) decrementing the buffer would make it leave the unsafe interval
- // 3) decrementing the buffer would yield a number below w_high and farther
- // away than the current number. In other words:
- // (buffer{-1} < w_high) && w_high - buffer{-1} > buffer - w_high
- // Instead of using the buffer directly we use its distance to too_high.
- // Conceptually rest ~= too_high - buffer
- // We need to do the following tests in this order to avoid over- and
- // underflows.
- ASSERT(rest <= unsafe_interval);
- while (rest < small_distance && // Negated condition 1
- unsafe_interval - rest >= ten_kappa && // Negated condition 2
- (rest + ten_kappa < small_distance || // buffer{-1} > w_high
- small_distance - rest >= rest + ten_kappa - small_distance)) {
- buffer[length - 1]--;
- rest += ten_kappa;
- }
-
- // We have approached w+ as much as possible. We now test if approaching w-
- // would require changing the buffer. If yes, then we have two possible
- // representations close to w, but we cannot decide which one is closer.
- if (rest < big_distance &&
- unsafe_interval - rest >= ten_kappa &&
- (rest + ten_kappa < big_distance ||
- big_distance - rest > rest + ten_kappa - big_distance)) {
- return false;
- }
-
- // Weeding test.
- // The safe interval is [too_low + 2 ulp; too_high - 2 ulp]
- // Since too_low = too_high - unsafe_interval this is equivalent to
- // [too_high - unsafe_interval + 4 ulp; too_high - 2 ulp]
- // Conceptually we have: rest ~= too_high - buffer
- return (2 * unit <= rest) && (rest <= unsafe_interval - 4 * unit);
-}
-
-
-// Rounds the buffer upwards if the result is closer to v by possibly adding
-// 1 to the buffer. If the precision of the calculation is not sufficient to
-// round correctly, return false.
-// The rounding might shift the whole buffer in which case the kappa is
-// adjusted. For example "99", kappa = 3 might become "10", kappa = 4.
-//
-// If 2*rest > ten_kappa then the buffer needs to be round up.
-// rest can have an error of +/- 1 unit. This function accounts for the
-// imprecision and returns false, if the rounding direction cannot be
-// unambiguously determined.
-//
-// Precondition: rest < ten_kappa.
-static bool RoundWeedCounted(Vector<char> buffer,
- int length,
- uint64_t rest,
- uint64_t ten_kappa,
- uint64_t unit,
- int* kappa) {
- ASSERT(rest < ten_kappa);
- // The following tests are done in a specific order to avoid overflows. They
- // will work correctly with any uint64 values of rest < ten_kappa and unit.
- //
- // If the unit is too big, then we don't know which way to round. For example
- // a unit of 50 means that the real number lies within rest +/- 50. If
- // 10^kappa == 40 then there is no way to tell which way to round.
- if (unit >= ten_kappa) return false;
- // Even if unit is just half the size of 10^kappa we are already completely
- // lost. (And after the previous test we know that the expression will not
- // over/underflow.)
- if (ten_kappa - unit <= unit) return false;
- // If 2 * (rest + unit) <= 10^kappa we can safely round down.
- if ((ten_kappa - rest > rest) && (ten_kappa - 2 * rest >= 2 * unit)) {
- return true;
- }
- // If 2 * (rest - unit) >= 10^kappa, then we can safely round up.
- if ((rest > unit) && (ten_kappa - (rest - unit) <= (rest - unit))) {
- // Increment the last digit recursively until we find a non '9' digit.
- buffer[length - 1]++;
- for (int i = length - 1; i > 0; --i) {
- if (buffer[i] != '0' + 10) break;
- buffer[i] = '0';
- buffer[i - 1]++;
- }
- // If the first digit is now '0'+ 10 we had a buffer with all '9's. With the
- // exception of the first digit all digits are now '0'. Simply switch the
- // first digit to '1' and adjust the kappa. Example: "99" becomes "10" and
- // the power (the kappa) is increased.
- if (buffer[0] == '0' + 10) {
- buffer[0] = '1';
- (*kappa) += 1;
- }
- return true;
- }
- return false;
-}
-
-// Returns the biggest power of ten that is less than or equal to the given
-// number. We furthermore receive the maximum number of bits 'number' has.
-//
-// Returns power == 10^(exponent_plus_one-1) such that
-// power <= number < power * 10.
-// If number_bits == 0 then 0^(0-1) is returned.
-// The number of bits must be <= 32.
-// Precondition: number < (1 << (number_bits + 1)).
-
-// Inspired by the method for finding an integer log base 10 from here:
-// http://graphics.stanford.edu/~seander/bithacks.html#IntegerLog10
-static unsigned int const kSmallPowersOfTen[] =
- {0, 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000,
- 1000000000};
-
-static void BiggestPowerTen(uint32_t number,
- int number_bits,
- uint32_t* power,
- int* exponent_plus_one) {
- ASSERT(number < (1u << (number_bits + 1)));
- // 1233/4096 is approximately 1/lg(10).
- int exponent_plus_one_guess = ((number_bits + 1) * 1233 >> 12);
- // We increment to skip over the first entry in the kPowersOf10 table.
- // Note: kPowersOf10[i] == 10^(i-1).
- exponent_plus_one_guess++;
- // We don't have any guarantees that 2^number_bits <= number.
- // TODO(floitsch): can we change the 'while' into an 'if'? We definitely see
- // number < (2^number_bits - 1), but I haven't encountered
- // number < (2^number_bits - 2) yet.
- while (number < kSmallPowersOfTen[exponent_plus_one_guess]) {
- exponent_plus_one_guess--;
- }
- *power = kSmallPowersOfTen[exponent_plus_one_guess];
- *exponent_plus_one = exponent_plus_one_guess;
-}
-
-// Generates the digits of input number w.
-// w is a floating-point number (DiyFp), consisting of a significand and an
-// exponent. Its exponent is bounded by kMinimalTargetExponent and
-// kMaximalTargetExponent.
-// Hence -60 <= w.e() <= -32.
-//
-// Returns false if it fails, in which case the generated digits in the buffer
-// should not be used.
-// Preconditions:
-// * low, w and high are correct up to 1 ulp (unit in the last place). That
-// is, their error must be less than a unit of their last digits.
-// * low.e() == w.e() == high.e()
-// * low < w < high, and taking into account their error: low~ <= high~
-// * kMinimalTargetExponent <= w.e() <= kMaximalTargetExponent
-// Postconditions: returns false if procedure fails.
-// otherwise:
-// * buffer is not null-terminated, but len contains the number of digits.
-// * buffer contains the shortest possible decimal digit-sequence
-// such that LOW < buffer * 10^kappa < HIGH, where LOW and HIGH are the
-// correct values of low and high (without their error).
-// * if more than one decimal representation gives the minimal number of
-// decimal digits then the one closest to W (where W is the correct value
-// of w) is chosen.
-// Remark: this procedure takes into account the imprecision of its input
-// numbers. If the precision is not enough to guarantee all the postconditions
-// then false is returned. This usually happens rarely (~0.5%).
-//
-// Say, for the sake of example, that
-// w.e() == -48, and w.f() == 0x1234567890abcdef
-// w's value can be computed by w.f() * 2^w.e()
-// We can obtain w's integral digits by simply shifting w.f() by -w.e().
-// -> w's integral part is 0x1234
-// w's fractional part is therefore 0x567890abcdef.
-// Printing w's integral part is easy (simply print 0x1234 in decimal).
-// In order to print its fraction we repeatedly multiply the fraction by 10 and
-// get each digit. Example the first digit after the point would be computed by
-// (0x567890abcdef * 10) >> 48. -> 3
-// The whole thing becomes slightly more complicated because we want to stop
-// once we have enough digits. That is, once the digits inside the buffer
-// represent 'w' we can stop. Everything inside the interval low - high
-// represents w. However we have to pay attention to low, high and w's
-// imprecision.
-static bool DigitGen(DiyFp low,
- DiyFp w,
- DiyFp high,
- Vector<char> buffer,
- int* length,
- int* kappa) {
- ASSERT(low.e() == w.e() && w.e() == high.e());
- ASSERT(low.f() + 1 <= high.f() - 1);
- ASSERT(kMinimalTargetExponent <= w.e() && w.e() <= kMaximalTargetExponent);
- // low, w and high are imprecise, but by less than one ulp (unit in the last
- // place).
- // If we remove (resp. add) 1 ulp from low (resp. high) we are certain that
- // the new numbers are outside of the interval we want the final
- // representation to lie in.
- // Inversely adding (resp. removing) 1 ulp from low (resp. high) would yield
- // numbers that are certain to lie in the interval. We will use this fact
- // later on.
- // We will now start by generating the digits within the uncertain
- // interval. Later we will weed out representations that lie outside the safe
- // interval and thus _might_ lie outside the correct interval.
- uint64_t unit = 1;
- DiyFp too_low = DiyFp(low.f() - unit, low.e());
- DiyFp too_high = DiyFp(high.f() + unit, high.e());
- // too_low and too_high are guaranteed to lie outside the interval we want the
- // generated number in.
- DiyFp unsafe_interval = DiyFp::Minus(too_high, too_low);
- // We now cut the input number into two parts: the integral digits and the
- // fractionals. We will not write any decimal separator though, but adapt
- // kappa instead.
- // Reminder: we are currently computing the digits (stored inside the buffer)
- // such that: too_low < buffer * 10^kappa < too_high
- // We use too_high for the digit_generation and stop as soon as possible.
- // If we stop early we effectively round down.
- DiyFp one = DiyFp(static_cast<uint64_t>(1) << -w.e(), w.e());
- // Division by one is a shift.
- uint32_t integrals = static_cast<uint32_t>(too_high.f() >> -one.e());
- // Modulo by one is an and.
- uint64_t fractionals = too_high.f() & (one.f() - 1);
- uint32_t divisor;
- int divisor_exponent_plus_one;
- BiggestPowerTen(integrals, DiyFp::kSignificandSize - (-one.e()),
- &divisor, &divisor_exponent_plus_one);
- *kappa = divisor_exponent_plus_one;
- *length = 0;
- // Loop invariant: buffer = too_high / 10^kappa (integer division)
- // The invariant holds for the first iteration: kappa has been initialized
- // with the divisor exponent + 1. And the divisor is the biggest power of ten
- // that is smaller than integrals.
- while (*kappa > 0) {
- int digit = integrals / divisor;
- buffer[*length] = '0' + digit;
- (*length)++;
- integrals %= divisor;
- (*kappa)--;
- // Note that kappa now equals the exponent of the divisor and that the
- // invariant thus holds again.
- uint64_t rest =
- (static_cast<uint64_t>(integrals) << -one.e()) + fractionals;
- // Invariant: too_high = buffer * 10^kappa + DiyFp(rest, one.e())
- // Reminder: unsafe_interval.e() == one.e()
- if (rest < unsafe_interval.f()) {
- // Rounding down (by not emitting the remaining digits) yields a number
- // that lies within the unsafe interval.
- return RoundWeed(buffer, *length, DiyFp::Minus(too_high, w).f(),
- unsafe_interval.f(), rest,
- static_cast<uint64_t>(divisor) << -one.e(), unit);
- }
- divisor /= 10;
- }
-
- // The integrals have been generated. We are at the point of the decimal
- // separator. In the following loop we simply multiply the remaining digits by
- // 10 and divide by one. We just need to pay attention to multiply associated
- // data (like the interval or 'unit'), too.
- // Note that the multiplication by 10 does not overflow, because w.e >= -60
- // and thus one.e >= -60.
- ASSERT(one.e() >= -60);
- ASSERT(fractionals < one.f());
- ASSERT(UINT64_2PART_C(0xFFFFFFFF, FFFFFFFF) / 10 >= one.f());
- while (true) {
- fractionals *= 10;
- unit *= 10;
- unsafe_interval.set_f(unsafe_interval.f() * 10);
- // Integer division by one.
- int digit = static_cast<int>(fractionals >> -one.e());
- buffer[*length] = '0' + digit;
- (*length)++;
- fractionals &= one.f() - 1; // Modulo by one.
- (*kappa)--;
- if (fractionals < unsafe_interval.f()) {
- return RoundWeed(buffer, *length, DiyFp::Minus(too_high, w).f() * unit,
- unsafe_interval.f(), fractionals, one.f(), unit);
- }
- }
-}
-
-
-
-// Generates (at most) requested_digits digits of input number w.
-// w is a floating-point number (DiyFp), consisting of a significand and an
-// exponent. Its exponent is bounded by kMinimalTargetExponent and
-// kMaximalTargetExponent.
-// Hence -60 <= w.e() <= -32.
-//
-// Returns false if it fails, in which case the generated digits in the buffer
-// should not be used.
-// Preconditions:
-// * w is correct up to 1 ulp (unit in the last place). That
-// is, its error must be strictly less than a unit of its last digit.
-// * kMinimalTargetExponent <= w.e() <= kMaximalTargetExponent
-//
-// Postconditions: returns false if procedure fails.
-// otherwise:
-// * buffer is not null-terminated, but length contains the number of
-// digits.
-// * the representation in buffer is the most precise representation of
-// requested_digits digits.
-// * buffer contains at most requested_digits digits of w. If there are less
-// than requested_digits digits then some trailing '0's have been removed.
-// * kappa is such that
-// w = buffer * 10^kappa + eps with |eps| < 10^kappa / 2.
-//
-// Remark: This procedure takes into account the imprecision of its input
-// numbers. If the precision is not enough to guarantee all the postconditions
-// then false is returned. This usually happens rarely, but the failure-rate
-// increases with higher requested_digits.
-static bool DigitGenCounted(DiyFp w,
- int requested_digits,
- Vector<char> buffer,
- int* length,
- int* kappa) {
- ASSERT(kMinimalTargetExponent <= w.e() && w.e() <= kMaximalTargetExponent);
- ASSERT(kMinimalTargetExponent >= -60);
- ASSERT(kMaximalTargetExponent <= -32);
- // w is assumed to have an error less than 1 unit. Whenever w is scaled we
- // also scale its error.
- uint64_t w_error = 1;
- // We cut the input number into two parts: the integral digits and the
- // fractional digits. We don't emit any decimal separator, but adapt kappa
- // instead. Example: instead of writing "1.2" we put "12" into the buffer and
- // increase kappa by 1.
- DiyFp one = DiyFp(static_cast<uint64_t>(1) << -w.e(), w.e());
- // Division by one is a shift.
- uint32_t integrals = static_cast<uint32_t>(w.f() >> -one.e());
- // Modulo by one is an and.
- uint64_t fractionals = w.f() & (one.f() - 1);
- uint32_t divisor;
- int divisor_exponent_plus_one;
- BiggestPowerTen(integrals, DiyFp::kSignificandSize - (-one.e()),
- &divisor, &divisor_exponent_plus_one);
- *kappa = divisor_exponent_plus_one;
- *length = 0;
-
- // Loop invariant: buffer = w / 10^kappa (integer division)
- // The invariant holds for the first iteration: kappa has been initialized
- // with the divisor exponent + 1. And the divisor is the biggest power of ten
- // that is smaller than 'integrals'.
- while (*kappa > 0) {
- int digit = integrals / divisor;
- buffer[*length] = '0' + digit;
- (*length)++;
- requested_digits--;
- integrals %= divisor;
- (*kappa)--;
- // Note that kappa now equals the exponent of the divisor and that the
- // invariant thus holds again.
- if (requested_digits == 0) break;
- divisor /= 10;
- }
-
- if (requested_digits == 0) {
- uint64_t rest =
- (static_cast<uint64_t>(integrals) << -one.e()) + fractionals;
- return RoundWeedCounted(buffer, *length, rest,
- static_cast<uint64_t>(divisor) << -one.e(), w_error,
- kappa);
- }
-
- // The integrals have been generated. We are at the point of the decimal
- // separator. In the following loop we simply multiply the remaining digits by
- // 10 and divide by one. We just need to pay attention to multiply associated
- // data (the 'unit'), too.
- // Note that the multiplication by 10 does not overflow, because w.e >= -60
- // and thus one.e >= -60.
- ASSERT(one.e() >= -60);
- ASSERT(fractionals < one.f());
- ASSERT(UINT64_2PART_C(0xFFFFFFFF, FFFFFFFF) / 10 >= one.f());
- while (requested_digits > 0 && fractionals > w_error) {
- fractionals *= 10;
- w_error *= 10;
- // Integer division by one.
- int digit = static_cast<int>(fractionals >> -one.e());
- buffer[*length] = '0' + digit;
- (*length)++;
- requested_digits--;
- fractionals &= one.f() - 1; // Modulo by one.
- (*kappa)--;
- }
- if (requested_digits != 0) return false;
- return RoundWeedCounted(buffer, *length, fractionals, one.f(), w_error,
- kappa);
-}
-
-
-// Provides a decimal representation of v.
-// Returns true if it succeeds, otherwise the result cannot be trusted.
-// There will be *length digits inside the buffer (not null-terminated).
-// If the function returns true then
-// v == (double) (buffer * 10^decimal_exponent).
-// The digits in the buffer are the shortest representation possible: no
-// 0.09999999999999999 instead of 0.1. The shorter representation will even be
-// chosen even if the longer one would be closer to v.
-// The last digit will be closest to the actual v. That is, even if several
-// digits might correctly yield 'v' when read again, the closest will be
-// computed.
-static bool Grisu3(double v,
- FastDtoaMode mode,
- Vector<char> buffer,
- int* length,
- int* decimal_exponent) {
- DiyFp w = Double(v).AsNormalizedDiyFp();
- // boundary_minus and boundary_plus are the boundaries between v and its
- // closest floating-point neighbors. Any number strictly between
- // boundary_minus and boundary_plus will round to v when convert to a double.
- // Grisu3 will never output representations that lie exactly on a boundary.
- DiyFp boundary_minus, boundary_plus;
- if (mode == FAST_DTOA_SHORTEST) {
- Double(v).NormalizedBoundaries(&boundary_minus, &boundary_plus);
- } else {
- ASSERT(mode == FAST_DTOA_SHORTEST_SINGLE);
- float single_v = static_cast<float>(v);
- Single(single_v).NormalizedBoundaries(&boundary_minus, &boundary_plus);
- }
- ASSERT(boundary_plus.e() == w.e());
- DiyFp ten_mk; // Cached power of ten: 10^-k
- int mk; // -k
- int ten_mk_minimal_binary_exponent =
- kMinimalTargetExponent - (w.e() + DiyFp::kSignificandSize);
- int ten_mk_maximal_binary_exponent =
- kMaximalTargetExponent - (w.e() + DiyFp::kSignificandSize);
- PowersOfTenCache::GetCachedPowerForBinaryExponentRange(
- ten_mk_minimal_binary_exponent,
- ten_mk_maximal_binary_exponent,
- &ten_mk, &mk);
- ASSERT((kMinimalTargetExponent <= w.e() + ten_mk.e() +
- DiyFp::kSignificandSize) &&
- (kMaximalTargetExponent >= w.e() + ten_mk.e() +
- DiyFp::kSignificandSize));
- // Note that ten_mk is only an approximation of 10^-k. A DiyFp only contains a
- // 64 bit significand and ten_mk is thus only precise up to 64 bits.
-
- // The DiyFp::Times procedure rounds its result, and ten_mk is approximated
- // too. The variable scaled_w (as well as scaled_boundary_minus/plus) are now
- // off by a small amount.
- // In fact: scaled_w - w*10^k < 1ulp (unit in the last place) of scaled_w.
- // In other words: let f = scaled_w.f() and e = scaled_w.e(), then
- // (f-1) * 2^e < w*10^k < (f+1) * 2^e
- DiyFp scaled_w = DiyFp::Times(w, ten_mk);
- ASSERT(scaled_w.e() ==
- boundary_plus.e() + ten_mk.e() + DiyFp::kSignificandSize);
- // In theory it would be possible to avoid some recomputations by computing
- // the difference between w and boundary_minus/plus (a power of 2) and to
- // compute scaled_boundary_minus/plus by subtracting/adding from
- // scaled_w. However the code becomes much less readable and the speed
- // enhancements are not terriffic.
- DiyFp scaled_boundary_minus = DiyFp::Times(boundary_minus, ten_mk);
- DiyFp scaled_boundary_plus = DiyFp::Times(boundary_plus, ten_mk);
-
- // DigitGen will generate the digits of scaled_w. Therefore we have
- // v == (double) (scaled_w * 10^-mk).
- // Set decimal_exponent == -mk and pass it to DigitGen. If scaled_w is not an
- // integer than it will be updated. For instance if scaled_w == 1.23 then
- // the buffer will be filled with "123" und the decimal_exponent will be
- // decreased by 2.
- int kappa;
- bool result = DigitGen(scaled_boundary_minus, scaled_w, scaled_boundary_plus,
- buffer, length, &kappa);
- *decimal_exponent = -mk + kappa;
- return result;
-}
-
-
-// The "counted" version of grisu3 (see above) only generates requested_digits
-// number of digits. This version does not generate the shortest representation,
-// and with enough requested digits 0.1 will at some point print as 0.9999999...
-// Grisu3 is too imprecise for real halfway cases (1.5 will not work) and
-// therefore the rounding strategy for halfway cases is irrelevant.
-static bool Grisu3Counted(double v,
- int requested_digits,
- Vector<char> buffer,
- int* length,
- int* decimal_exponent) {
- DiyFp w = Double(v).AsNormalizedDiyFp();
- DiyFp ten_mk; // Cached power of ten: 10^-k
- int mk; // -k
- int ten_mk_minimal_binary_exponent =
- kMinimalTargetExponent - (w.e() + DiyFp::kSignificandSize);
- int ten_mk_maximal_binary_exponent =
- kMaximalTargetExponent - (w.e() + DiyFp::kSignificandSize);
- PowersOfTenCache::GetCachedPowerForBinaryExponentRange(
- ten_mk_minimal_binary_exponent,
- ten_mk_maximal_binary_exponent,
- &ten_mk, &mk);
- ASSERT((kMinimalTargetExponent <= w.e() + ten_mk.e() +
- DiyFp::kSignificandSize) &&
- (kMaximalTargetExponent >= w.e() + ten_mk.e() +
- DiyFp::kSignificandSize));
- // Note that ten_mk is only an approximation of 10^-k. A DiyFp only contains a
- // 64 bit significand and ten_mk is thus only precise up to 64 bits.
-
- // The DiyFp::Times procedure rounds its result, and ten_mk is approximated
- // too. The variable scaled_w (as well as scaled_boundary_minus/plus) are now
- // off by a small amount.
- // In fact: scaled_w - w*10^k < 1ulp (unit in the last place) of scaled_w.
- // In other words: let f = scaled_w.f() and e = scaled_w.e(), then
- // (f-1) * 2^e < w*10^k < (f+1) * 2^e
- DiyFp scaled_w = DiyFp::Times(w, ten_mk);
-
- // We now have (double) (scaled_w * 10^-mk).
- // DigitGen will generate the first requested_digits digits of scaled_w and
- // return together with a kappa such that scaled_w ~= buffer * 10^kappa. (It
- // will not always be exactly the same since DigitGenCounted only produces a
- // limited number of digits.)
- int kappa;
- bool result = DigitGenCounted(scaled_w, requested_digits,
- buffer, length, &kappa);
- *decimal_exponent = -mk + kappa;
- return result;
-}
-
-
-bool FastDtoa(double v,
- FastDtoaMode mode,
- int requested_digits,
- Vector<char> buffer,
- int* length,
- int* decimal_point) {
- ASSERT(v > 0);
- ASSERT(!Double(v).IsSpecial());
-
- bool result = false;
- int decimal_exponent = 0;
- switch (mode) {
- case FAST_DTOA_SHORTEST:
- case FAST_DTOA_SHORTEST_SINGLE:
- result = Grisu3(v, mode, buffer, length, &decimal_exponent);
- break;
- case FAST_DTOA_PRECISION:
- result = Grisu3Counted(v, requested_digits,
- buffer, length, &decimal_exponent);
- break;
- default:
- UNREACHABLE();
- }
- if (result) {
- *decimal_point = *length + decimal_exponent;
- buffer[*length] = '\0';
- }
- return result;
-}
-
-} // namespace double_conversion
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/double-conversion/fast-dtoa.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/double-conversion/fast-dtoa.h b/ext/kenlm/util/double-conversion/fast-dtoa.h
deleted file mode 100644
index 5f1e8ee..0000000
--- a/ext/kenlm/util/double-conversion/fast-dtoa.h
+++ /dev/null
@@ -1,88 +0,0 @@
-// Copyright 2010 the V8 project authors. All rights reserved.
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following
-// disclaimer in the documentation and/or other materials provided
-// with the distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived
-// from this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#ifndef DOUBLE_CONVERSION_FAST_DTOA_H_
-#define DOUBLE_CONVERSION_FAST_DTOA_H_
-
-#include "utils.h"
-
-namespace double_conversion {
-
-enum FastDtoaMode {
- // Computes the shortest representation of the given input. The returned
- // result will be the most accurate number of this length. Longer
- // representations might be more accurate.
- FAST_DTOA_SHORTEST,
- // Same as FAST_DTOA_SHORTEST but for single-precision floats.
- FAST_DTOA_SHORTEST_SINGLE,
- // Computes a representation where the precision (number of digits) is
- // given as input. The precision is independent of the decimal point.
- FAST_DTOA_PRECISION
-};
-
-// FastDtoa will produce at most kFastDtoaMaximalLength digits. This does not
-// include the terminating '\0' character.
-static const int kFastDtoaMaximalLength = 17;
-// Same for single-precision numbers.
-static const int kFastDtoaMaximalSingleLength = 9;
-
-// Provides a decimal representation of v.
-// The result should be interpreted as buffer * 10^(point - length).
-//
-// Precondition:
-// * v must be a strictly positive finite double.
-//
-// Returns true if it succeeds, otherwise the result can not be trusted.
-// There will be *length digits inside the buffer followed by a null terminator.
-// If the function returns true and mode equals
-// - FAST_DTOA_SHORTEST, then
-// the parameter requested_digits is ignored.
-// The result satisfies
-// v == (double) (buffer * 10^(point - length)).
-// The digits in the buffer are the shortest representation possible. E.g.
-// if 0.099999999999 and 0.1 represent the same double then "1" is returned
-// with point = 0.
-// The last digit will be closest to the actual v. That is, even if several
-// digits might correctly yield 'v' when read again, the buffer will contain
-// the one closest to v.
-// - FAST_DTOA_PRECISION, then
-// the buffer contains requested_digits digits.
-// the difference v - (buffer * 10^(point-length)) is closest to zero for
-// all possible representations of requested_digits digits.
-// If there are two values that are equally close, then FastDtoa returns
-// false.
-// For both modes the buffer must be large enough to hold the result.
-bool FastDtoa(double d,
- FastDtoaMode mode,
- int requested_digits,
- Vector<char> buffer,
- int* length,
- int* decimal_point);
-
-} // namespace double_conversion
-
-#endif // DOUBLE_CONVERSION_FAST_DTOA_H_
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/double-conversion/fixed-dtoa.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/double-conversion/fixed-dtoa.cc b/ext/kenlm/util/double-conversion/fixed-dtoa.cc
deleted file mode 100644
index 7c1a952..0000000
--- a/ext/kenlm/util/double-conversion/fixed-dtoa.cc
+++ /dev/null
@@ -1,402 +0,0 @@
-// Copyright 2010 the V8 project authors. All rights reserved.
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following
-// disclaimer in the documentation and/or other materials provided
-// with the distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived
-// from this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#include <cmath>
-
-#include "fixed-dtoa.h"
-#include "ieee.h"
-
-namespace double_conversion {
-
-// Represents a 128bit type. This class should be replaced by a native type on
-// platforms that support 128bit integers.
-class UInt128 {
- public:
- UInt128() : high_bits_(0), low_bits_(0) { }
- UInt128(uint64_t high, uint64_t low) : high_bits_(high), low_bits_(low) { }
-
- void Multiply(uint32_t multiplicand) {
- uint64_t accumulator;
-
- accumulator = (low_bits_ & kMask32) * multiplicand;
- uint32_t part = static_cast<uint32_t>(accumulator & kMask32);
- accumulator >>= 32;
- accumulator = accumulator + (low_bits_ >> 32) * multiplicand;
- low_bits_ = (accumulator << 32) + part;
- accumulator >>= 32;
- accumulator = accumulator + (high_bits_ & kMask32) * multiplicand;
- part = static_cast<uint32_t>(accumulator & kMask32);
- accumulator >>= 32;
- accumulator = accumulator + (high_bits_ >> 32) * multiplicand;
- high_bits_ = (accumulator << 32) + part;
- ASSERT((accumulator >> 32) == 0);
- }
-
- void Shift(int shift_amount) {
- ASSERT(-64 <= shift_amount && shift_amount <= 64);
- if (shift_amount == 0) {
- return;
- } else if (shift_amount == -64) {
- high_bits_ = low_bits_;
- low_bits_ = 0;
- } else if (shift_amount == 64) {
- low_bits_ = high_bits_;
- high_bits_ = 0;
- } else if (shift_amount <= 0) {
- high_bits_ <<= -shift_amount;
- high_bits_ += low_bits_ >> (64 + shift_amount);
- low_bits_ <<= -shift_amount;
- } else {
- low_bits_ >>= shift_amount;
- low_bits_ += high_bits_ << (64 - shift_amount);
- high_bits_ >>= shift_amount;
- }
- }
-
- // Modifies *this to *this MOD (2^power).
- // Returns *this DIV (2^power).
- int DivModPowerOf2(int power) {
- if (power >= 64) {
- int result = static_cast<int>(high_bits_ >> (power - 64));
- high_bits_ -= static_cast<uint64_t>(result) << (power - 64);
- return result;
- } else {
- uint64_t part_low = low_bits_ >> power;
- uint64_t part_high = high_bits_ << (64 - power);
- int result = static_cast<int>(part_low + part_high);
- high_bits_ = 0;
- low_bits_ -= part_low << power;
- return result;
- }
- }
-
- bool IsZero() const {
- return high_bits_ == 0 && low_bits_ == 0;
- }
-
- int BitAt(int position) {
- if (position >= 64) {
- return static_cast<int>(high_bits_ >> (position - 64)) & 1;
- } else {
- return static_cast<int>(low_bits_ >> position) & 1;
- }
- }
-
- private:
- static const uint64_t kMask32 = 0xFFFFFFFF;
- // Value == (high_bits_ << 64) + low_bits_
- uint64_t high_bits_;
- uint64_t low_bits_;
-};
-
-
-static const int kDoubleSignificandSize = 53; // Includes the hidden bit.
-
-
-static void FillDigits32FixedLength(uint32_t number, int requested_length,
- Vector<char> buffer, int* length) {
- for (int i = requested_length - 1; i >= 0; --i) {
- buffer[(*length) + i] = '0' + number % 10;
- number /= 10;
- }
- *length += requested_length;
-}
-
-
-static void FillDigits32(uint32_t number, Vector<char> buffer, int* length) {
- int number_length = 0;
- // We fill the digits in reverse order and exchange them afterwards.
- while (number != 0) {
- int digit = number % 10;
- number /= 10;
- buffer[(*length) + number_length] = '0' + digit;
- number_length++;
- }
- // Exchange the digits.
- int i = *length;
- int j = *length + number_length - 1;
- while (i < j) {
- char tmp = buffer[i];
- buffer[i] = buffer[j];
- buffer[j] = tmp;
- i++;
- j--;
- }
- *length += number_length;
-}
-
-
-static void FillDigits64FixedLength(uint64_t number, int requested_length,
- Vector<char> buffer, int* length) {
- const uint32_t kTen7 = 10000000;
- // For efficiency cut the number into 3 uint32_t parts, and print those.
- uint32_t part2 = static_cast<uint32_t>(number % kTen7);
- number /= kTen7;
- uint32_t part1 = static_cast<uint32_t>(number % kTen7);
- uint32_t part0 = static_cast<uint32_t>(number / kTen7);
-
- FillDigits32FixedLength(part0, 3, buffer, length);
- FillDigits32FixedLength(part1, 7, buffer, length);
- FillDigits32FixedLength(part2, 7, buffer, length);
-}
-
-
-static void FillDigits64(uint64_t number, Vector<char> buffer, int* length) {
- const uint32_t kTen7 = 10000000;
- // For efficiency cut the number into 3 uint32_t parts, and print those.
- uint32_t part2 = static_cast<uint32_t>(number % kTen7);
- number /= kTen7;
- uint32_t part1 = static_cast<uint32_t>(number % kTen7);
- uint32_t part0 = static_cast<uint32_t>(number / kTen7);
-
- if (part0 != 0) {
- FillDigits32(part0, buffer, length);
- FillDigits32FixedLength(part1, 7, buffer, length);
- FillDigits32FixedLength(part2, 7, buffer, length);
- } else if (part1 != 0) {
- FillDigits32(part1, buffer, length);
- FillDigits32FixedLength(part2, 7, buffer, length);
- } else {
- FillDigits32(part2, buffer, length);
- }
-}
-
-
-static void RoundUp(Vector<char> buffer, int* length, int* decimal_point) {
- // An empty buffer represents 0.
- if (*length == 0) {
- buffer[0] = '1';
- *decimal_point = 1;
- *length = 1;
- return;
- }
- // Round the last digit until we either have a digit that was not '9' or until
- // we reached the first digit.
- buffer[(*length) - 1]++;
- for (int i = (*length) - 1; i > 0; --i) {
- if (buffer[i] != '0' + 10) {
- return;
- }
- buffer[i] = '0';
- buffer[i - 1]++;
- }
- // If the first digit is now '0' + 10, we would need to set it to '0' and add
- // a '1' in front. However we reach the first digit only if all following
- // digits had been '9' before rounding up. Now all trailing digits are '0' and
- // we simply switch the first digit to '1' and update the decimal-point
- // (indicating that the point is now one digit to the right).
- if (buffer[0] == '0' + 10) {
- buffer[0] = '1';
- (*decimal_point)++;
- }
-}
-
-
-// The given fractionals number represents a fixed-point number with binary
-// point at bit (-exponent).
-// Preconditions:
-// -128 <= exponent <= 0.
-// 0 <= fractionals * 2^exponent < 1
-// The buffer holds the result.
-// The function will round its result. During the rounding-process digits not
-// generated by this function might be updated, and the decimal-point variable
-// might be updated. If this function generates the digits 99 and the buffer
-// already contained "199" (thus yielding a buffer of "19999") then a
-// rounding-up will change the contents of the buffer to "20000".
-static void FillFractionals(uint64_t fractionals, int exponent,
- int fractional_count, Vector<char> buffer,
- int* length, int* decimal_point) {
- ASSERT(-128 <= exponent && exponent <= 0);
- // 'fractionals' is a fixed-point number, with binary point at bit
- // (-exponent). Inside the function the non-converted remainder of fractionals
- // is a fixed-point number, with binary point at bit 'point'.
- if (-exponent <= 64) {
- // One 64 bit number is sufficient.
- ASSERT(fractionals >> 56 == 0);
- int point = -exponent;
- for (int i = 0; i < fractional_count; ++i) {
- if (fractionals == 0) break;
- // Instead of multiplying by 10 we multiply by 5 and adjust the point
- // location. This way the fractionals variable will not overflow.
- // Invariant at the beginning of the loop: fractionals < 2^point.
- // Initially we have: point <= 64 and fractionals < 2^56
- // After each iteration the point is decremented by one.
- // Note that 5^3 = 125 < 128 = 2^7.
- // Therefore three iterations of this loop will not overflow fractionals
- // (even without the subtraction at the end of the loop body). At this
- // time point will satisfy point <= 61 and therefore fractionals < 2^point
- // and any further multiplication of fractionals by 5 will not overflow.
- fractionals *= 5;
- point--;
- int digit = static_cast<int>(fractionals >> point);
- buffer[*length] = '0' + digit;
- (*length)++;
- fractionals -= static_cast<uint64_t>(digit) << point;
- }
- // If the first bit after the point is set we have to round up.
- if (((fractionals >> (point - 1)) & 1) == 1) {
- RoundUp(buffer, length, decimal_point);
- }
- } else { // We need 128 bits.
- ASSERT(64 < -exponent && -exponent <= 128);
- UInt128 fractionals128 = UInt128(fractionals, 0);
- fractionals128.Shift(-exponent - 64);
- int point = 128;
- for (int i = 0; i < fractional_count; ++i) {
- if (fractionals128.IsZero()) break;
- // As before: instead of multiplying by 10 we multiply by 5 and adjust the
- // point location.
- // This multiplication will not overflow for the same reasons as before.
- fractionals128.Multiply(5);
- point--;
- int digit = fractionals128.DivModPowerOf2(point);
- buffer[*length] = '0' + digit;
- (*length)++;
- }
- if (fractionals128.BitAt(point - 1) == 1) {
- RoundUp(buffer, length, decimal_point);
- }
- }
-}
-
-
-// Removes leading and trailing zeros.
-// If leading zeros are removed then the decimal point position is adjusted.
-static void TrimZeros(Vector<char> buffer, int* length, int* decimal_point) {
- while (*length > 0 && buffer[(*length) - 1] == '0') {
- (*length)--;
- }
- int first_non_zero = 0;
- while (first_non_zero < *length && buffer[first_non_zero] == '0') {
- first_non_zero++;
- }
- if (first_non_zero != 0) {
- for (int i = first_non_zero; i < *length; ++i) {
- buffer[i - first_non_zero] = buffer[i];
- }
- *length -= first_non_zero;
- *decimal_point -= first_non_zero;
- }
-}
-
-
-bool FastFixedDtoa(double v,
- int fractional_count,
- Vector<char> buffer,
- int* length,
- int* decimal_point) {
- const uint32_t kMaxUInt32 = 0xFFFFFFFF;
- uint64_t significand = Double(v).Significand();
- int exponent = Double(v).Exponent();
- // v = significand * 2^exponent (with significand a 53bit integer).
- // If the exponent is larger than 20 (i.e. we may have a 73bit number) then we
- // don't know how to compute the representation. 2^73 ~= 9.5*10^21.
- // If necessary this limit could probably be increased, but we don't need
- // more.
- if (exponent > 20) return false;
- if (fractional_count > 20) return false;
- *length = 0;
- // At most kDoubleSignificandSize bits of the significand are non-zero.
- // Given a 64 bit integer we have 11 0s followed by 53 potentially non-zero
- // bits: 0..11*..0xxx..53*..xx
- if (exponent + kDoubleSignificandSize > 64) {
- // The exponent must be > 11.
- //
- // We know that v = significand * 2^exponent.
- // And the exponent > 11.
- // We simplify the task by dividing v by 10^17.
- // The quotient delivers the first digits, and the remainder fits into a 64
- // bit number.
- // Dividing by 10^17 is equivalent to dividing by 5^17*2^17.
- const uint64_t kFive17 = UINT64_2PART_C(0xB1, A2BC2EC5); // 5^17
- uint64_t divisor = kFive17;
- int divisor_power = 17;
- uint64_t dividend = significand;
- uint32_t quotient;
- uint64_t remainder;
- // Let v = f * 2^e with f == significand and e == exponent.
- // Then need q (quotient) and r (remainder) as follows:
- // v = q * 10^17 + r
- // f * 2^e = q * 10^17 + r
- // f * 2^e = q * 5^17 * 2^17 + r
- // If e > 17 then
- // f * 2^(e-17) = q * 5^17 + r/2^17
- // else
- // f = q * 5^17 * 2^(17-e) + r/2^e
- if (exponent > divisor_power) {
- // We only allow exponents of up to 20 and therefore (17 - e) <= 3
- dividend <<= exponent - divisor_power;
- quotient = static_cast<uint32_t>(dividend / divisor);
- remainder = (dividend % divisor) << divisor_power;
- } else {
- divisor <<= divisor_power - exponent;
- quotient = static_cast<uint32_t>(dividend / divisor);
- remainder = (dividend % divisor) << exponent;
- }
- FillDigits32(quotient, buffer, length);
- FillDigits64FixedLength(remainder, divisor_power, buffer, length);
- *decimal_point = *length;
- } else if (exponent >= 0) {
- // 0 <= exponent <= 11
- significand <<= exponent;
- FillDigits64(significand, buffer, length);
- *decimal_point = *length;
- } else if (exponent > -kDoubleSignificandSize) {
- // We have to cut the number.
- uint64_t integrals = significand >> -exponent;
- uint64_t fractionals = significand - (integrals << -exponent);
- if (integrals > kMaxUInt32) {
- FillDigits64(integrals, buffer, length);
- } else {
- FillDigits32(static_cast<uint32_t>(integrals), buffer, length);
- }
- *decimal_point = *length;
- FillFractionals(fractionals, exponent, fractional_count,
- buffer, length, decimal_point);
- } else if (exponent < -128) {
- // This configuration (with at most 20 digits) means that all digits must be
- // 0.
- ASSERT(fractional_count <= 20);
- buffer[0] = '\0';
- *length = 0;
- *decimal_point = -fractional_count;
- } else {
- *decimal_point = 0;
- FillFractionals(significand, exponent, fractional_count,
- buffer, length, decimal_point);
- }
- TrimZeros(buffer, length, decimal_point);
- buffer[*length] = '\0';
- if ((*length) == 0) {
- // The string is empty and the decimal_point thus has no importance. Mimick
- // Gay's dtoa and and set it to -fractional_count.
- *decimal_point = -fractional_count;
- }
- return true;
-}
-
-} // namespace double_conversion
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/double-conversion/fixed-dtoa.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/double-conversion/fixed-dtoa.h b/ext/kenlm/util/double-conversion/fixed-dtoa.h
deleted file mode 100644
index 3bdd08e..0000000
--- a/ext/kenlm/util/double-conversion/fixed-dtoa.h
+++ /dev/null
@@ -1,56 +0,0 @@
-// Copyright 2010 the V8 project authors. All rights reserved.
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following
-// disclaimer in the documentation and/or other materials provided
-// with the distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived
-// from this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#ifndef DOUBLE_CONVERSION_FIXED_DTOA_H_
-#define DOUBLE_CONVERSION_FIXED_DTOA_H_
-
-#include "utils.h"
-
-namespace double_conversion {
-
-// Produces digits necessary to print a given number with
-// 'fractional_count' digits after the decimal point.
-// The buffer must be big enough to hold the result plus one terminating null
-// character.
-//
-// The produced digits might be too short in which case the caller has to fill
-// the gaps with '0's.
-// Example: FastFixedDtoa(0.001, 5, ...) is allowed to return buffer = "1", and
-// decimal_point = -2.
-// Halfway cases are rounded towards +/-Infinity (away from 0). The call
-// FastFixedDtoa(0.15, 2, ...) thus returns buffer = "2", decimal_point = 0.
-// The returned buffer may contain digits that would be truncated from the
-// shortest representation of the input.
-//
-// This method only works for some parameters. If it can't handle the input it
-// returns false. The output is null-terminated when the function succeeds.
-bool FastFixedDtoa(double v, int fractional_count,
- Vector<char> buffer, int* length, int* decimal_point);
-
-} // namespace double_conversion
-
-#endif // DOUBLE_CONVERSION_FIXED_DTOA_H_
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/double-conversion/ieee.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/double-conversion/ieee.h b/ext/kenlm/util/double-conversion/ieee.h
deleted file mode 100644
index 839dc47..0000000
--- a/ext/kenlm/util/double-conversion/ieee.h
+++ /dev/null
@@ -1,398 +0,0 @@
-// Copyright 2012 the V8 project authors. All rights reserved.
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following
-// disclaimer in the documentation and/or other materials provided
-// with the distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived
-// from this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#ifndef DOUBLE_CONVERSION_DOUBLE_H_
-#define DOUBLE_CONVERSION_DOUBLE_H_
-
-#include "diy-fp.h"
-
-namespace double_conversion {
-
-// We assume that doubles and uint64_t have the same endianness.
-static uint64_t double_to_uint64(double d) { return BitCast<uint64_t>(d); }
-static double uint64_to_double(uint64_t d64) { return BitCast<double>(d64); }
-static uint32_t float_to_uint32(float f) { return BitCast<uint32_t>(f); }
-static float uint32_to_float(uint32_t d32) { return BitCast<float>(d32); }
-
-// Helper functions for doubles.
-class Double {
- public:
- static const uint64_t kSignMask = UINT64_2PART_C(0x80000000, 00000000);
- static const uint64_t kExponentMask = UINT64_2PART_C(0x7FF00000, 00000000);
- static const uint64_t kSignificandMask = UINT64_2PART_C(0x000FFFFF, FFFFFFFF);
- static const uint64_t kHiddenBit = UINT64_2PART_C(0x00100000, 00000000);
- static const int kPhysicalSignificandSize = 52; // Excludes the hidden bit.
- static const int kSignificandSize = 53;
-
- Double() : d64_(0) {}
- explicit Double(double d) : d64_(double_to_uint64(d)) {}
- explicit Double(uint64_t d64) : d64_(d64) {}
- explicit Double(DiyFp diy_fp)
- : d64_(DiyFpToUint64(diy_fp)) {}
-
- // The value encoded by this Double must be greater or equal to +0.0.
- // It must not be special (infinity, or NaN).
- DiyFp AsDiyFp() const {
- ASSERT(Sign() > 0);
- ASSERT(!IsSpecial());
- return DiyFp(Significand(), Exponent());
- }
-
- // The value encoded by this Double must be strictly greater than 0.
- DiyFp AsNormalizedDiyFp() const {
- ASSERT(value() > 0.0);
- uint64_t f = Significand();
- int e = Exponent();
-
- // The current double could be a denormal.
- while ((f & kHiddenBit) == 0) {
- f <<= 1;
- e--;
- }
- // Do the final shifts in one go.
- f <<= DiyFp::kSignificandSize - kSignificandSize;
- e -= DiyFp::kSignificandSize - kSignificandSize;
- return DiyFp(f, e);
- }
-
- // Returns the double's bit as uint64.
- uint64_t AsUint64() const {
- return d64_;
- }
-
- // Returns the next greater double. Returns +infinity on input +infinity.
- double NextDouble() const {
- if (d64_ == kInfinity) return Double(kInfinity).value();
- if (Sign() < 0 && Significand() == 0) {
- // -0.0
- return 0.0;
- }
- if (Sign() < 0) {
- return Double(d64_ - 1).value();
- } else {
- return Double(d64_ + 1).value();
- }
- }
-
- double PreviousDouble() const {
- if (d64_ == (kInfinity | kSignMask)) return -Double::Infinity();
- if (Sign() < 0) {
- return Double(d64_ + 1).value();
- } else {
- if (Significand() == 0) return -0.0;
- return Double(d64_ - 1).value();
- }
- }
-
- int Exponent() const {
- if (IsDenormal()) return kDenormalExponent;
-
- uint64_t d64 = AsUint64();
- int biased_e =
- static_cast<int>((d64 & kExponentMask) >> kPhysicalSignificandSize);
- return biased_e - kExponentBias;
- }
-
- uint64_t Significand() const {
- uint64_t d64 = AsUint64();
- uint64_t significand = d64 & kSignificandMask;
- if (!IsDenormal()) {
- return significand + kHiddenBit;
- } else {
- return significand;
- }
- }
-
- // Returns true if the double is a denormal.
- bool IsDenormal() const {
- uint64_t d64 = AsUint64();
- return (d64 & kExponentMask) == 0;
- }
-
- // We consider denormals not to be special.
- // Hence only Infinity and NaN are special.
- bool IsSpecial() const {
- uint64_t d64 = AsUint64();
- return (d64 & kExponentMask) == kExponentMask;
- }
-
- bool IsNan() const {
- uint64_t d64 = AsUint64();
- return ((d64 & kExponentMask) == kExponentMask) &&
- ((d64 & kSignificandMask) != 0);
- }
-
- bool IsInfinite() const {
- uint64_t d64 = AsUint64();
- return ((d64 & kExponentMask) == kExponentMask) &&
- ((d64 & kSignificandMask) == 0);
- }
-
- int Sign() const {
- uint64_t d64 = AsUint64();
- return (d64 & kSignMask) == 0? 1: -1;
- }
-
- // Precondition: the value encoded by this Double must be greater or equal
- // than +0.0.
- DiyFp UpperBoundary() const {
- ASSERT(Sign() > 0);
- return DiyFp(Significand() * 2 + 1, Exponent() - 1);
- }
-
- // Computes the two boundaries of this.
- // The bigger boundary (m_plus) is normalized. The lower boundary has the same
- // exponent as m_plus.
- // Precondition: the value encoded by this Double must be greater than 0.
- void NormalizedBoundaries(DiyFp* out_m_minus, DiyFp* out_m_plus) const {
- ASSERT(value() > 0.0);
- DiyFp v = this->AsDiyFp();
- DiyFp m_plus = DiyFp::Normalize(DiyFp((v.f() << 1) + 1, v.e() - 1));
- DiyFp m_minus;
- if (LowerBoundaryIsCloser()) {
- m_minus = DiyFp((v.f() << 2) - 1, v.e() - 2);
- } else {
- m_minus = DiyFp((v.f() << 1) - 1, v.e() - 1);
- }
- m_minus.set_f(m_minus.f() << (m_minus.e() - m_plus.e()));
- m_minus.set_e(m_plus.e());
- *out_m_plus = m_plus;
- *out_m_minus = m_minus;
- }
-
- bool LowerBoundaryIsCloser() const {
- // The boundary is closer if the significand is of the form f == 2^p-1 then
- // the lower boundary is closer.
- // Think of v = 1000e10 and v- = 9999e9.
- // Then the boundary (== (v - v-)/2) is not just at a distance of 1e9 but
- // at a distance of 1e8.
- // The only exception is for the smallest normal: the largest denormal is
- // at the same distance as its successor.
- // Note: denormals have the same exponent as the smallest normals.
- bool physical_significand_is_zero = ((AsUint64() & kSignificandMask) == 0);
- return physical_significand_is_zero && (Exponent() != kDenormalExponent);
- }
-
- double value() const { return uint64_to_double(d64_); }
-
- // Returns the significand size for a given order of magnitude.
- // If v = f*2^e with 2^p-1 <= f <= 2^p then p+e is v's order of magnitude.
- // This function returns the number of significant binary digits v will have
- // once it's encoded into a double. In almost all cases this is equal to
- // kSignificandSize. The only exceptions are denormals. They start with
- // leading zeroes and their effective significand-size is hence smaller.
- static int SignificandSizeForOrderOfMagnitude(int order) {
- if (order >= (kDenormalExponent + kSignificandSize)) {
- return kSignificandSize;
- }
- if (order <= kDenormalExponent) return 0;
- return order - kDenormalExponent;
- }
-
- static double Infinity() {
- return Double(kInfinity).value();
- }
-
- static double NaN() {
- return Double(kNaN).value();
- }
-
- private:
- static const int kExponentBias = 0x3FF + kPhysicalSignificandSize;
- static const int kDenormalExponent = -kExponentBias + 1;
- static const int kMaxExponent = 0x7FF - kExponentBias;
- static const uint64_t kInfinity = UINT64_2PART_C(0x7FF00000, 00000000);
- static const uint64_t kNaN = UINT64_2PART_C(0x7FF80000, 00000000);
-
- const uint64_t d64_;
-
- static uint64_t DiyFpToUint64(DiyFp diy_fp) {
- uint64_t significand = diy_fp.f();
- int exponent = diy_fp.e();
- while (significand > kHiddenBit + kSignificandMask) {
- significand >>= 1;
- exponent++;
- }
- if (exponent >= kMaxExponent) {
- return kInfinity;
- }
- if (exponent < kDenormalExponent) {
- return 0;
- }
- while (exponent > kDenormalExponent && (significand & kHiddenBit) == 0) {
- significand <<= 1;
- exponent--;
- }
- uint64_t biased_exponent;
- if (exponent == kDenormalExponent && (significand & kHiddenBit) == 0) {
- biased_exponent = 0;
- } else {
- biased_exponent = static_cast<uint64_t>(exponent + kExponentBias);
- }
- return (significand & kSignificandMask) |
- (biased_exponent << kPhysicalSignificandSize);
- }
-};
-
-class Single {
- public:
- static const uint32_t kSignMask = 0x80000000;
- static const uint32_t kExponentMask = 0x7F800000;
- static const uint32_t kSignificandMask = 0x007FFFFF;
- static const uint32_t kHiddenBit = 0x00800000;
- static const int kPhysicalSignificandSize = 23; // Excludes the hidden bit.
- static const int kSignificandSize = 24;
-
- Single() : d32_(0) {}
- explicit Single(float f) : d32_(float_to_uint32(f)) {}
- explicit Single(uint32_t d32) : d32_(d32) {}
-
- // The value encoded by this Single must be greater or equal to +0.0.
- // It must not be special (infinity, or NaN).
- DiyFp AsDiyFp() const {
- ASSERT(Sign() > 0);
- ASSERT(!IsSpecial());
- return DiyFp(Significand(), Exponent());
- }
-
- // Returns the single's bit as uint64.
- uint32_t AsUint32() const {
- return d32_;
- }
-
- int Exponent() const {
- if (IsDenormal()) return kDenormalExponent;
-
- uint32_t d32 = AsUint32();
- int biased_e =
- static_cast<int>((d32 & kExponentMask) >> kPhysicalSignificandSize);
- return biased_e - kExponentBias;
- }
-
- uint32_t Significand() const {
- uint32_t d32 = AsUint32();
- uint32_t significand = d32 & kSignificandMask;
- if (!IsDenormal()) {
- return significand + kHiddenBit;
- } else {
- return significand;
- }
- }
-
- // Returns true if the single is a denormal.
- bool IsDenormal() const {
- uint32_t d32 = AsUint32();
- return (d32 & kExponentMask) == 0;
- }
-
- // We consider denormals not to be special.
- // Hence only Infinity and NaN are special.
- bool IsSpecial() const {
- uint32_t d32 = AsUint32();
- return (d32 & kExponentMask) == kExponentMask;
- }
-
- bool IsNan() const {
- uint32_t d32 = AsUint32();
- return ((d32 & kExponentMask) == kExponentMask) &&
- ((d32 & kSignificandMask) != 0);
- }
-
- bool IsInfinite() const {
- uint32_t d32 = AsUint32();
- return ((d32 & kExponentMask) == kExponentMask) &&
- ((d32 & kSignificandMask) == 0);
- }
-
- int Sign() const {
- uint32_t d32 = AsUint32();
- return (d32 & kSignMask) == 0? 1: -1;
- }
-
- // Computes the two boundaries of this.
- // The bigger boundary (m_plus) is normalized. The lower boundary has the same
- // exponent as m_plus.
- // Precondition: the value encoded by this Single must be greater than 0.
- void NormalizedBoundaries(DiyFp* out_m_minus, DiyFp* out_m_plus) const {
- ASSERT(value() > 0.0);
- DiyFp v = this->AsDiyFp();
- DiyFp m_plus = DiyFp::Normalize(DiyFp((v.f() << 1) + 1, v.e() - 1));
- DiyFp m_minus;
- if (LowerBoundaryIsCloser()) {
- m_minus = DiyFp((v.f() << 2) - 1, v.e() - 2);
- } else {
- m_minus = DiyFp((v.f() << 1) - 1, v.e() - 1);
- }
- m_minus.set_f(m_minus.f() << (m_minus.e() - m_plus.e()));
- m_minus.set_e(m_plus.e());
- *out_m_plus = m_plus;
- *out_m_minus = m_minus;
- }
-
- // Precondition: the value encoded by this Single must be greater or equal
- // than +0.0.
- DiyFp UpperBoundary() const {
- ASSERT(Sign() > 0);
- return DiyFp(Significand() * 2 + 1, Exponent() - 1);
- }
-
- bool LowerBoundaryIsCloser() const {
- // The boundary is closer if the significand is of the form f == 2^p-1 then
- // the lower boundary is closer.
- // Think of v = 1000e10 and v- = 9999e9.
- // Then the boundary (== (v - v-)/2) is not just at a distance of 1e9 but
- // at a distance of 1e8.
- // The only exception is for the smallest normal: the largest denormal is
- // at the same distance as its successor.
- // Note: denormals have the same exponent as the smallest normals.
- bool physical_significand_is_zero = ((AsUint32() & kSignificandMask) == 0);
- return physical_significand_is_zero && (Exponent() != kDenormalExponent);
- }
-
- float value() const { return uint32_to_float(d32_); }
-
- static float Infinity() {
- return Single(kInfinity).value();
- }
-
- static float NaN() {
- return Single(kNaN).value();
- }
-
- private:
- static const int kExponentBias = 0x7F + kPhysicalSignificandSize;
- static const int kDenormalExponent = -kExponentBias + 1;
- static const int kMaxExponent = 0xFF - kExponentBias;
- static const uint32_t kInfinity = 0x7F800000;
- static const uint32_t kNaN = 0x7FC00000;
-
- const uint32_t d32_;
-};
-
-} // namespace double_conversion
-
-#endif // DOUBLE_CONVERSION_DOUBLE_H_
[16/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/search.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/search.c b/ext/kenlm/jam-files/engine/search.c
deleted file mode 100644
index b2beada..0000000
--- a/ext/kenlm/jam-files/engine/search.c
+++ /dev/null
@@ -1,274 +0,0 @@
-/*
- * Copyright 1993-2002 Christopher Seiwald and Perforce Software, Inc.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#include "jam.h"
-#include "search.h"
-
-#include "compile.h"
-#include "filesys.h"
-#include "hash.h"
-#include "lists.h"
-#include "object.h"
-#include "pathsys.h"
-#include "strings.h"
-#include "timestamp.h"
-#include "variable.h"
-
-#include <string.h>
-
-
-typedef struct _binding
-{
- OBJECT * binding;
- OBJECT * target;
-} BINDING;
-
-static struct hash * explicit_bindings = 0;
-
-
-void call_bind_rule( OBJECT * target_, OBJECT * boundname_ )
-{
- LIST * const bind_rule = var_get( root_module(), constant_BINDRULE );
- if ( !list_empty( bind_rule ) )
- {
- OBJECT * target = object_copy( target_ );
- OBJECT * boundname = object_copy( boundname_ );
- if ( boundname && target )
- {
- /* Prepare the argument list. */
- FRAME frame[ 1 ];
- frame_init( frame );
-
- /* First argument is the target name. */
- lol_add( frame->args, list_new( target ) );
-
- lol_add( frame->args, list_new( boundname ) );
- if ( lol_get( frame->args, 1 ) )
- {
- OBJECT * rulename = list_front( bind_rule );
- list_free( evaluate_rule( bindrule( rulename, root_module() ), rulename, frame ) );
- }
-
- /* Clean up */
- frame_free( frame );
- }
- else
- {
- if ( boundname )
- object_free( boundname );
- if ( target )
- object_free( target );
- }
- }
-}
-
-/* Records the binding of a target with an explicit LOCATE. */
-void set_explicit_binding( OBJECT * target, OBJECT * locate )
-{
- OBJECT * boundname;
- OBJECT * key;
- PATHNAME f[ 1 ];
- string buf[ 1 ];
- int found;
- BINDING * ba;
-
- if ( !explicit_bindings )
- explicit_bindings = hashinit( sizeof( BINDING ), "explicitly specified "
- "locations" );
-
- string_new( buf );
-
- /* Parse the filename. */
- path_parse( object_str( target ), f );
-
- /* Ignore the grist. */
- f->f_grist.ptr = 0;
- f->f_grist.len = 0;
-
- /* Root the target path at the given location. */
- f->f_root.ptr = object_str( locate );
- f->f_root.len = strlen( object_str( locate ) );
-
- path_build( f, buf );
- boundname = object_new( buf->value );
- if ( DEBUG_SEARCH )
- printf( "explicit locate %s: %s\n", object_str( target ), buf->value );
- string_free( buf );
- key = path_as_key( boundname );
- object_free( boundname );
-
- ba = (BINDING *)hash_insert( explicit_bindings, key, &found );
- if ( !found )
- {
- ba->binding = key;
- ba->target = target;
- }
- else
- object_free( key );
-}
-
-/*
- * search.c - find a target along $(SEARCH) or $(LOCATE).
- *
- * First, check if LOCATE is set. If so, use it to determine the location of
- * target and return, regardless of whether anything exists at that location.
- *
- * Second, examine all directories in SEARCH. If the file exists there or there
- * is another target with the same name already placed at this location via the
- * LOCATE setting, stop and return the location. In case of a previous target,
- * return its name via the 'another_target' argument.
- *
- * This behaviour allows handling dependencies on generated files.
- *
- * If caller does not expect that the target is generated, 0 can be passed as
- * 'another_target'.
- */
-
-OBJECT * search( OBJECT * target, timestamp * const time,
- OBJECT * * another_target, int const file )
-{
- PATHNAME f[ 1 ];
- LIST * varlist;
- string buf[ 1 ];
- int found = 0;
- OBJECT * boundname = 0;
-
- if ( another_target )
- *another_target = 0;
-
- if ( !explicit_bindings )
- explicit_bindings = hashinit( sizeof( BINDING ), "explicitly specified "
- "locations" );
-
- string_new( buf );
-
- /* Parse the filename. */
- path_parse( object_str( target ), f );
-
- f->f_grist.ptr = 0;
- f->f_grist.len = 0;
-
- varlist = var_get( root_module(), constant_LOCATE );
- if ( !list_empty( varlist ) )
- {
- OBJECT * key;
- f->f_root.ptr = object_str( list_front( varlist ) );
- f->f_root.len = strlen( object_str( list_front( varlist ) ) );
-
- path_build( f, buf );
-
- if ( DEBUG_SEARCH )
- printf( "locate %s: %s\n", object_str( target ), buf->value );
-
- key = object_new( buf->value );
- timestamp_from_path( time, key );
- object_free( key );
- found = 1;
- }
- else if ( varlist = var_get( root_module(), constant_SEARCH ),
- !list_empty( varlist ) )
- {
- LISTITER iter = list_begin( varlist );
- LISTITER const end = list_end( varlist );
- for ( ; iter != end; iter = list_next( iter ) )
- {
- BINDING * ba;
- file_info_t * ff;
- OBJECT * key;
- OBJECT * test_path;
-
- f->f_root.ptr = object_str( list_item( iter ) );
- f->f_root.len = strlen( object_str( list_item( iter ) ) );
-
- string_truncate( buf, 0 );
- path_build( f, buf );
-
- if ( DEBUG_SEARCH )
- printf( "search %s: %s\n", object_str( target ), buf->value );
-
- test_path = object_new( buf->value );
- key = path_as_key( test_path );
- object_free( test_path );
- ff = file_query( key );
- timestamp_from_path( time, key );
-
- if ( ( ba = (BINDING *)hash_find( explicit_bindings, key ) ) )
- {
- if ( DEBUG_SEARCH )
- printf(" search %s: found explicitly located target %s\n",
- object_str( target ), object_str( ba->target ) );
- if ( another_target )
- *another_target = ba->target;
- found = 1;
- object_free( key );
- break;
- }
- else if ( ff )
- {
- if ( !file || ff->is_file )
- {
- found = 1;
- object_free( key );
- break;
- }
- }
- object_free( key );
- }
- }
-
- if ( !found )
- {
- /* Look for the obvious. */
- /* This is a questionable move. Should we look in the obvious place if
- * SEARCH is set?
- */
- OBJECT * key;
-
- f->f_root.ptr = 0;
- f->f_root.len = 0;
-
- string_truncate( buf, 0 );
- path_build( f, buf );
-
- if ( DEBUG_SEARCH )
- printf( "search %s: %s\n", object_str( target ), buf->value );
-
- key = object_new( buf->value );
- timestamp_from_path( time, key );
- object_free( key );
- }
-
- boundname = object_new( buf->value );
- string_free( buf );
-
- /* Prepare a call to BINDRULE if the variable is set. */
- call_bind_rule( target, boundname );
-
- return boundname;
-}
-
-
-static void free_binding( void * xbinding, void * data )
-{
- object_free( ( (BINDING *)xbinding )->binding );
-}
-
-
-void search_done( void )
-{
- if ( explicit_bindings )
- {
- hashenumerate( explicit_bindings, free_binding, 0 );
- hashdone( explicit_bindings );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/search.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/search.h b/ext/kenlm/jam-files/engine/search.h
deleted file mode 100644
index 7e74f79..0000000
--- a/ext/kenlm/jam-files/engine/search.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Copyright 1993, 1995 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/*
- * search.h - find a target along $(SEARCH) or $(LOCATE)
- */
-
-#ifndef SEARCH_SW20111118_H
-#define SEARCH_SW20111118_H
-
-#include "object.h"
-#include "timestamp.h"
-
-void set_explicit_binding( OBJECT * target, OBJECT * locate );
-OBJECT * search( OBJECT * target, timestamp * const time,
- OBJECT * * another_target, int const file );
-void search_done( void );
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/strings.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/strings.c b/ext/kenlm/jam-files/engine/strings.c
deleted file mode 100644
index 3d3e19b..0000000
--- a/ext/kenlm/jam-files/engine/strings.c
+++ /dev/null
@@ -1,223 +0,0 @@
-/* Copyright David Abrahams 2004. Distributed under the Boost */
-/* Software License, Version 1.0. (See accompanying */
-/* file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) */
-
-#include "jam.h"
-#include "strings.h"
-
-#include <assert.h>
-#include <stdlib.h>
-#include <string.h>
-
-
-#ifndef NDEBUG
-# define JAM_STRING_MAGIC ((char)0xcf)
-# define JAM_STRING_MAGIC_SIZE 4
-static void assert_invariants( string * self )
-{
- int i;
-
- if ( self->value == 0 )
- {
- assert( self->size == 0 );
- assert( self->capacity == 0 );
- assert( self->opt[ 0 ] == 0 );
- return;
- }
-
- assert( self->size < self->capacity );
- assert( ( self->capacity <= sizeof( self->opt ) ) == ( self->value == self->opt ) );
- assert( self->value[ self->size ] == 0 );
- /* String objects modified manually after construction to contain embedded
- * '\0' characters are considered structurally valid.
- */
- assert( strlen( self->value ) <= self->size );
-
- for ( i = 0; i < 4; ++i )
- {
- assert( self->magic[ i ] == JAM_STRING_MAGIC );
- assert( self->value[ self->capacity + i ] == JAM_STRING_MAGIC );
- }
-}
-#else
-# define JAM_STRING_MAGIC_SIZE 0
-# define assert_invariants(x) do {} while (0)
-#endif
-
-
-void string_new( string * s )
-{
- s->value = s->opt;
- s->size = 0;
- s->capacity = sizeof( s->opt );
- s->opt[ 0 ] = 0;
-#ifndef NDEBUG
- memset( s->magic, JAM_STRING_MAGIC, sizeof( s->magic ) );
-#endif
- assert_invariants( s );
-}
-
-
-void string_free( string * s )
-{
- assert_invariants( s );
- if ( s->value != s->opt )
- BJAM_FREE( s->value );
- string_new( s );
-}
-
-
-static void string_reserve_internal( string * self, size_t capacity )
-{
- if ( self->value == self->opt )
- {
- self->value = (char *)BJAM_MALLOC_ATOMIC( capacity +
- JAM_STRING_MAGIC_SIZE );
- self->value[ 0 ] = 0;
- strncat( self->value, self->opt, sizeof(self->opt) );
- assert( strlen( self->value ) <= self->capacity && "Regression test" );
- }
- else
- {
- self->value = (char *)BJAM_REALLOC( self->value, capacity +
- JAM_STRING_MAGIC_SIZE );
- }
-#ifndef NDEBUG
- memcpy( self->value + capacity, self->magic, JAM_STRING_MAGIC_SIZE );
-#endif
- self->capacity = capacity;
-}
-
-
-void string_reserve( string * self, size_t capacity )
-{
- assert_invariants( self );
- if ( capacity <= self->capacity )
- return;
- string_reserve_internal( self, capacity );
- assert_invariants( self );
-}
-
-
-static void extend_full( string * self, char const * start, char const * finish )
-{
- size_t new_size = self->capacity + ( finish - start );
- size_t new_capacity = self->capacity;
- size_t old_size = self->capacity;
- while ( new_capacity < new_size + 1)
- new_capacity <<= 1;
- string_reserve_internal( self, new_capacity );
- memcpy( self->value + old_size, start, new_size - old_size );
- self->value[ new_size ] = 0;
- self->size = new_size;
-}
-
-static void maybe_reserve( string * self, size_t new_size )
-{
- size_t capacity = self->capacity;
- if ( capacity <= new_size )
- {
- size_t new_capacity = capacity;
- while ( new_capacity <= new_size )
- new_capacity <<= 1;
- string_reserve_internal( self, new_capacity );
- }
-}
-
-
-void string_append( string * self, char const * rhs )
-{
- size_t rhs_size = strlen( rhs );
- size_t new_size = self->size + rhs_size;
- assert_invariants( self );
-
- maybe_reserve( self, new_size );
-
- memcpy( self->value + self->size, rhs, rhs_size + 1 );
- self->size = new_size;
-
- assert_invariants( self );
-}
-
-
-void string_append_range( string * self, char const * start, char const * finish )
-{
- size_t rhs_size = finish - start;
- size_t new_size = self->size + rhs_size;
- assert_invariants( self );
-
- maybe_reserve( self, new_size );
-
- memcpy( self->value + self->size, start, rhs_size );
- self->size = new_size;
- self->value[ new_size ] = 0;
-
- assert_invariants( self );
-}
-
-
-void string_copy( string * s, char const * rhs )
-{
- string_new( s );
- string_append( s, rhs );
-}
-
-void string_truncate( string * self, size_t n )
-{
- assert_invariants( self );
- assert( n <= self->capacity );
- self->value[ self->size = n ] = 0;
- assert_invariants( self );
-}
-
-
-void string_pop_back( string * self )
-{
- string_truncate( self, self->size - 1 );
-}
-
-
-void string_push_back( string * self, char x )
-{
- string_append_range( self, &x, &x + 1 );
-}
-
-
-char string_back( string * self )
-{
- assert_invariants( self );
- return self->value[ self->size - 1 ];
-}
-
-
-#ifndef NDEBUG
-void string_unit_test()
-{
- {
- string s[ 1 ];
- int i;
- int const limit = sizeof( s->opt ) * 2 + 2;
- string_new( s );
- assert( s->value == s->opt );
- for ( i = 0; i < limit; ++i )
- {
- string_push_back( s, (char)( i + 1 ) );
- assert( s->size == i + 1 );
- }
- assert( s->size == limit );
- assert( s->value != s->opt );
- for ( i = 0; i < limit; ++i )
- assert( s->value[ i ] == (char)( i + 1 ) );
- string_free( s );
- }
-
- {
- char * const original = " \n\t\v Foo \r\n\v \tBar\n\n\r\r\t\n\v\t \t";
- string copy[ 1 ];
- string_copy( copy, original );
- assert( !strcmp( copy->value, original ) );
- assert( copy->size == strlen( original ) );
- string_free( copy );
- }
-}
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/strings.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/strings.h b/ext/kenlm/jam-files/engine/strings.h
deleted file mode 100644
index 749f287..0000000
--- a/ext/kenlm/jam-files/engine/strings.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Copyright 2004. David Abrahams
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#ifndef STRINGS_DWA20011024_H
-#define STRINGS_DWA20011024_H
-
-#include <stddef.h>
-
-typedef struct string
-{
- char * value;
- unsigned long size;
- unsigned long capacity;
- char opt[ 32 ];
-#ifndef NDEBUG
- char magic[ 4 ];
-#endif
-} string;
-
-void string_new( string * );
-void string_copy( string *, char const * );
-void string_free( string * );
-void string_append( string *, char const * );
-void string_append_range( string *, char const *, char const * );
-void string_push_back( string * s, char x );
-void string_reserve( string *, size_t );
-void string_truncate( string *, size_t );
-void string_pop_back( string * );
-char string_back( string * );
-void string_unit_test();
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/subst.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/subst.c b/ext/kenlm/jam-files/engine/subst.c
deleted file mode 100644
index a5fcee0..0000000
--- a/ext/kenlm/jam-files/engine/subst.c
+++ /dev/null
@@ -1,116 +0,0 @@
-#include "jam.h"
-#include "subst.h"
-
-#include "builtins.h"
-#include "frames.h"
-#include "hash.h"
-#include "lists.h"
-
-#include <stddef.h>
-
-
-typedef struct regex_entry
-{
- OBJECT * pattern;
- regexp * regex;
-} regex_entry;
-
-static struct hash * regex_hash;
-
-
-regexp * regex_compile( OBJECT * pattern )
-{
- int found;
- regex_entry * e ;
-
- if ( !regex_hash )
- regex_hash = hashinit( sizeof( regex_entry ), "regex" );
-
- e = (regex_entry *)hash_insert( regex_hash, pattern, &found );
- if ( !found )
- {
- e->pattern = object_copy( pattern );
- e->regex = regcomp( (char *)pattern );
- }
-
- return e->regex;
-}
-
-
-LIST * builtin_subst( FRAME * frame, int flags )
-{
- LIST * result = L0;
- LIST * const arg1 = lol_get( frame->args, 0 );
- LISTITER iter = list_begin( arg1 );
- LISTITER const end = list_end( arg1 );
-
- if ( iter != end && list_next( iter ) != end && list_next( list_next( iter )
- ) != end )
- {
- char const * const source = object_str( list_item( iter ) );
- OBJECT * const pattern = list_item( list_next( iter ) );
- regexp * const repat = regex_compile( pattern );
-
- if ( regexec( repat, (char *)source) )
- {
- LISTITER subst = list_next( iter );
-
- while ( ( subst = list_next( subst ) ) != end )
- {
-#define BUFLEN 4096
- char buf[ BUFLEN + 1 ];
- char const * in = object_str( list_item( subst ) );
- char * out = buf;
-
- for ( ; *in && out < buf + BUFLEN; ++in )
- {
- if ( *in == '\\' || *in == '$' )
- {
- ++in;
- if ( *in == 0 )
- break;
- if ( *in >= '0' && *in <= '9' )
- {
- unsigned int const n = *in - '0';
- size_t const srclen = repat->endp[ n ] -
- repat->startp[ n ];
- size_t const remaining = buf + BUFLEN - out;
- size_t const len = srclen < remaining
- ? srclen
- : remaining;
- memcpy( out, repat->startp[ n ], len );
- out += len;
- continue;
- }
- /* fall through and copy the next character */
- }
- *out++ = *in;
- }
- *out = 0;
-
- result = list_push_back( result, object_new( buf ) );
-#undef BUFLEN
- }
- }
- }
-
- return result;
-}
-
-
-static void free_regex( void * xregex, void * data )
-{
- regex_entry * const regex = (regex_entry *)xregex;
- object_free( regex->pattern );
- BJAM_FREE( regex->regex );
-}
-
-
-void regex_done()
-{
- if ( regex_hash )
- {
- hashenumerate( regex_hash, free_regex, (void *)0 );
- hashdone( regex_hash );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/subst.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/subst.h b/ext/kenlm/jam-files/engine/subst.h
deleted file mode 100644
index 7dc09a6..0000000
--- a/ext/kenlm/jam-files/engine/subst.h
+++ /dev/null
@@ -1,14 +0,0 @@
-/* Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#ifndef SUBST_JG20120722_H
-#define SUBST_JG20120722_H
-
-#include "object.h"
-#include "regexp.h"
-
-regexp * regex_compile( OBJECT * pattern );
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/timestamp.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/timestamp.c b/ext/kenlm/jam-files/engine/timestamp.c
deleted file mode 100644
index 0d01698..0000000
--- a/ext/kenlm/jam-files/engine/timestamp.c
+++ /dev/null
@@ -1,262 +0,0 @@
-/*
- * Copyright 1993-2002 Christopher Seiwald and Perforce Software, Inc.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * timestamp.c - get the timestamp of a file or archive member
- *
- * External routines:
- * timestamp_from_path() - return timestamp for a path, if present
- * timestamp_done() - free timestamp tables
- *
- * Internal routines:
- * time_enter() - internal worker callback for scanning archives &
- * directories
- * free_timestamps() - worker function for freeing timestamp table contents
- */
-
-#include "jam.h"
-#include "timestamp.h"
-
-#include "filesys.h"
-#include "hash.h"
-#include "object.h"
-#include "pathsys.h"
-#include "strings.h"
-
-
-/*
- * BINDING - all known files
- */
-
-typedef struct _binding
-{
- OBJECT * name;
- short flags;
-
-#define BIND_SCANNED 0x01 /* if directory or arch, has been scanned */
-
- short progress;
-
-#define BIND_INIT 0 /* never seen */
-#define BIND_NOENTRY 1 /* timestamp requested but file never found */
-#define BIND_SPOTTED 2 /* file found but not timed yet */
-#define BIND_MISSING 3 /* file found but can not get timestamp */
-#define BIND_FOUND 4 /* file found and time stamped */
-
- /* update time - cleared if the there is nothing to bind */
- timestamp time;
-} BINDING;
-
-static struct hash * bindhash = 0;
-
-static void time_enter( void *, OBJECT *, int const found,
- timestamp const * const );
-
-static char * time_progress[] =
-{
- "INIT",
- "NOENTRY",
- "SPOTTED",
- "MISSING",
- "FOUND"
-};
-
-
-#ifdef OS_NT
-/*
- * timestamp_from_filetime() - Windows FILETIME --> timestamp conversion
- *
- * Lifted shamelessly from the CPython implementation.
- */
-
-void timestamp_from_filetime( timestamp * const t, FILETIME const * const ft )
-{
- /* Seconds between 1.1.1601 and 1.1.1970 */
- static __int64 const secs_between_epochs = 11644473600;
-
- /* We can not simply cast and dereference a FILETIME, since it might not be
- * aligned properly. __int64 type variables are expected to be aligned to an
- * 8 byte boundary while FILETIME structures may be aligned to any 4 byte
- * boundary. Using an incorrectly aligned __int64 variable may cause a
- * performance penalty on some platforms or even exceptions on others
- * (documented on MSDN).
- */
- __int64 in;
- memcpy( &in, ft, sizeof( in ) );
-
- /* FILETIME resolution: 100ns. */
- timestamp_init( t, (time_t)( ( in / 10000000 ) - secs_between_epochs ),
- (int)( in % 10000000 ) * 100 );
-}
-#endif /* OS_NT */
-
-
-void timestamp_clear( timestamp * const time )
-{
- time->secs = time->nsecs = 0;
-}
-
-
-int timestamp_cmp( timestamp const * const lhs, timestamp const * const rhs )
-{
- return lhs->secs == rhs->secs
- ? lhs->nsecs - rhs->nsecs
- : lhs->secs - rhs->secs;
-}
-
-
-void timestamp_copy( timestamp * const target, timestamp const * const source )
-{
- target->secs = source->secs;
- target->nsecs = source->nsecs;
-}
-
-
-void timestamp_current( timestamp * const t )
-{
-#ifdef OS_NT
- /* GetSystemTimeAsFileTime()'s resolution seems to be about 15 ms on Windows
- * XP and under a millisecond on Windows 7.
- */
- FILETIME ft;
- GetSystemTimeAsFileTime( &ft );
- timestamp_from_filetime( t, &ft );
-#else /* OS_NT */
- timestamp_init( t, time( 0 ), 0 );
-#endif /* OS_NT */
-}
-
-
-int timestamp_empty( timestamp const * const time )
-{
- return !time->secs && !time->nsecs;
-}
-
-
-/*
- * timestamp_from_path() - return timestamp for a path, if present
- */
-
-void timestamp_from_path( timestamp * const time, OBJECT * const path )
-{
- PROFILE_ENTER( timestamp );
-
- PATHNAME f1;
- PATHNAME f2;
- int found;
- BINDING * b;
- string buf[ 1 ];
-
-
- if ( file_time( path, time ) < 0 )
- timestamp_clear( time );
-
- PROFILE_EXIT( timestamp );
-}
-
-
-void timestamp_init( timestamp * const time, time_t const secs, int const nsecs
- )
-{
- time->secs = secs;
- time->nsecs = nsecs;
-}
-
-
-void timestamp_max( timestamp * const max, timestamp const * const lhs,
- timestamp const * const rhs )
-{
- if ( timestamp_cmp( lhs, rhs ) > 0 )
- timestamp_copy( max, lhs );
- else
- timestamp_copy( max, rhs );
-}
-
-
-static char const * timestamp_formatstr( timestamp const * const time,
- char const * const format )
-{
- static char result1[ 500 ];
- static char result2[ 500 ];
- strftime( result1, sizeof( result1 ) / sizeof( *result1 ), format, gmtime(
- &time->secs ) );
- sprintf( result2, result1, time->nsecs );
- return result2;
-}
-
-
-char const * timestamp_str( timestamp const * const time )
-{
- return timestamp_formatstr( time, "%Y-%m-%d %H:%M:%S.%%09d +0000" );
-}
-
-
-char const * timestamp_timestr( timestamp const * const time )
-{
- return timestamp_formatstr( time, "%H:%M:%S.%%09d" );
-}
-
-
-/*
- * time_enter() - internal worker callback for scanning archives & directories
- */
-
-static void time_enter( void * closure, OBJECT * target, int const found,
- timestamp const * const time )
-{
- int item_found;
- BINDING * b;
- struct hash * const bindhash = (struct hash *)closure;
-
- target = path_as_key( target );
-
- b = (BINDING *)hash_insert( bindhash, target, &item_found );
- if ( !item_found )
- {
- b->name = object_copy( target );
- b->flags = 0;
- }
-
- timestamp_copy( &b->time, time );
- b->progress = found ? BIND_FOUND : BIND_SPOTTED;
-
- if ( DEBUG_BINDSCAN )
- printf( "time ( %s ) : %s\n", object_str( target ), time_progress[
- b->progress ] );
-
- object_free( target );
-}
-
-
-/*
- * free_timestamps() - worker function for freeing timestamp table contents
- */
-
-static void free_timestamps( void * xbinding, void * data )
-{
- object_free( ( (BINDING *)xbinding )->name );
-}
-
-
-/*
- * timestamp_done() - free timestamp tables
- */
-
-void timestamp_done()
-{
- if ( bindhash )
- {
- hashenumerate( bindhash, free_timestamps, 0 );
- hashdone( bindhash );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/timestamp.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/timestamp.h b/ext/kenlm/jam-files/engine/timestamp.h
deleted file mode 100644
index aaf1310..0000000
--- a/ext/kenlm/jam-files/engine/timestamp.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright 1993, 1995 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/*
- * timestamp.h - get the timestamp of a file or archive member
- */
-
-#ifndef TIMESTAMP_H_SW_2011_11_18
-#define TIMESTAMP_H_SW_2011_11_18
-
-#include "object.h"
-
-#ifdef OS_NT
-# define WIN32_LEAN_AND_MEAN
-# include <windows.h>
-#endif
-
-#include <time.h>
-
-typedef struct timestamp
-{
- time_t secs;
- int nsecs;
-} timestamp;
-
-void timestamp_clear( timestamp * const );
-int timestamp_cmp( timestamp const * const lhs, timestamp const * const rhs );
-void timestamp_copy( timestamp * const target, timestamp const * const source );
-void timestamp_current( timestamp * const );
-int timestamp_empty( timestamp const * const );
-void timestamp_from_path( timestamp * const, OBJECT * const path );
-void timestamp_init( timestamp * const, time_t const secs, int const nsecs );
-void timestamp_max( timestamp * const max, timestamp const * const lhs,
- timestamp const * const rhs );
-char const * timestamp_str( timestamp const * const );
-char const * timestamp_timestr( timestamp const * const );
-
-#ifdef OS_NT
-void timestamp_from_filetime( timestamp * const, FILETIME const * const );
-#endif
-
-void timestamp_done();
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/variable.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/variable.c b/ext/kenlm/jam-files/engine/variable.c
deleted file mode 100644
index 2c292fb..0000000
--- a/ext/kenlm/jam-files/engine/variable.c
+++ /dev/null
@@ -1,345 +0,0 @@
-/*
- * Copyright 1993, 2000 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Copyright 2005 Reece H. Dunn.
- * Copyright 2005 Rene Rivera.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * variable.c - handle Jam multi-element variables.
- *
- * External routines:
- *
- * var_defines() - load a bunch of variable=value settings
- * var_get() - get value of a user defined symbol
- * var_set() - set a variable in jam's user defined symbol table.
- * var_swap() - swap a variable's value with the given one
- * var_done() - free variable tables
- *
- * Internal routines:
- *
- * var_enter() - make new var symbol table entry, returning var ptr
- * var_dump() - dump a variable to stdout
- */
-
-#include "jam.h"
-#include "variable.h"
-
-#include "filesys.h"
-#include "hash.h"
-#include "modules.h"
-#include "parse.h"
-#include "pathsys.h"
-#include "strings.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-
-
-/*
- * VARIABLE - a user defined multi-value variable
- */
-
-typedef struct _variable VARIABLE ;
-
-struct _variable
-{
- OBJECT * symbol;
- LIST * value;
-};
-
-static LIST * * var_enter( struct module_t *, OBJECT * symbol );
-static void var_dump( OBJECT * symbol, LIST * value, char * what );
-
-
-/*
- * var_defines() - load a bunch of variable=value settings
- *
- * If preprocess is false, take the value verbatim.
- *
- * Otherwise, if the variable value is enclosed in quotes, strip the quotes.
- * Otherwise, if variable name ends in PATH, split value at :'s.
- * Otherwise, split the value at blanks.
- */
-
-void var_defines( struct module_t * module, char * const * e, int preprocess )
-{
- string buf[ 1 ];
-
- string_new( buf );
-
- for ( ; *e; ++e )
- {
- char * val;
-
- if ( ( val = strchr( *e, '=' ) )
-#if defined( OS_MAC )
- /* On the mac (MPW), the var=val is actually var\0val */
- /* Think different. */
- || ( val = *e + strlen( *e ) )
-#endif
- )
- {
- LIST * l = L0;
- size_t const len = strlen( val + 1 );
- int const quoted = ( val[ 1 ] == '"' ) && ( val[ len ] == '"' ) &&
- ( len > 1 );
-
- if ( quoted && preprocess )
- {
- string_append_range( buf, val + 2, val + len );
- l = list_push_back( l, object_new( buf->value ) );
- string_truncate( buf, 0 );
- }
- else
- {
- char * p;
- char * pp;
- char split =
-#if defined( OPT_NO_EXTERNAL_VARIABLE_SPLIT )
- '\0'
-#elif defined( OS_MAC )
- ','
-#else
- ' '
-#endif
- ;
-
- /* Split *PATH at :'s, not spaces. */
- if ( val - 4 >= *e )
- {
- if ( !strncmp( val - 4, "PATH", 4 ) ||
- !strncmp( val - 4, "Path", 4 ) ||
- !strncmp( val - 4, "path", 4 ) )
- split = SPLITPATH;
- }
-
- /* Do the split. */
- for
- (
- pp = val + 1;
- preprocess && ( ( p = strchr( pp, split ) ) != 0 );
- pp = p + 1
- )
- {
- string_append_range( buf, pp, p );
- l = list_push_back( l, object_new( buf->value ) );
- string_truncate( buf, 0 );
- }
-
- l = list_push_back( l, object_new( pp ) );
- }
-
- /* Get name. */
- string_append_range( buf, *e, val );
- {
- OBJECT * const varname = object_new( buf->value );
- var_set( module, varname, l, VAR_SET );
- object_free( varname );
- }
- string_truncate( buf, 0 );
- }
- }
- string_free( buf );
-}
-
-
-/* Last returned variable value saved so we may clear it in var_done(). */
-static LIST * saved_var = L0;
-
-
-/*
- * var_get() - get value of a user defined symbol
- *
- * Returns NULL if symbol unset.
- */
-
-LIST * var_get( struct module_t * module, OBJECT * symbol )
-{
- LIST * result = L0;
-#ifdef OPT_AT_FILES
- /* Some "fixed" variables... */
- if ( object_equal( symbol, constant_TMPDIR ) )
- {
- list_free( saved_var );
- result = saved_var = list_new( object_new( path_tmpdir()->value ) );
- }
- else if ( object_equal( symbol, constant_TMPNAME ) )
- {
- list_free( saved_var );
- result = saved_var = list_new( path_tmpnam() );
- }
- else if ( object_equal( symbol, constant_TMPFILE ) )
- {
- list_free( saved_var );
- result = saved_var = list_new( path_tmpfile() );
- }
- else if ( object_equal( symbol, constant_STDOUT ) )
- {
- list_free( saved_var );
- result = saved_var = list_new( object_copy( constant_STDOUT ) );
- }
- else if ( object_equal( symbol, constant_STDERR ) )
- {
- list_free( saved_var );
- result = saved_var = list_new( object_copy( constant_STDERR ) );
- }
- else
-#endif
- {
- VARIABLE * v;
- int n;
-
- if ( ( n = module_get_fixed_var( module, symbol ) ) != -1 )
- {
- if ( DEBUG_VARGET )
- var_dump( symbol, module->fixed_variables[ n ], "get" );
- result = module->fixed_variables[ n ];
- }
- else if ( module->variables && ( v = (VARIABLE *)hash_find(
- module->variables, symbol ) ) )
- {
- if ( DEBUG_VARGET )
- var_dump( v->symbol, v->value, "get" );
- result = v->value;
- }
- }
- return result;
-}
-
-
-LIST * var_get_and_clear_raw( module_t * module, OBJECT * symbol )
-{
- LIST * result = L0;
- VARIABLE * v;
-
- if ( module->variables && ( v = (VARIABLE *)hash_find( module->variables,
- symbol ) ) )
- {
- result = v->value;
- v->value = L0;
- }
-
- return result;
-}
-
-
-/*
- * var_set() - set a variable in Jam's user defined symbol table
- *
- * 'flag' controls the relationship between new and old values of the variable:
- * SET replaces the old with the new; APPEND appends the new to the old; DEFAULT
- * only uses the new if the variable was previously unset.
- *
- * Copies symbol. Takes ownership of value.
- */
-
-void var_set( struct module_t * module, OBJECT * symbol, LIST * value, int flag
- )
-{
- LIST * * v = var_enter( module, symbol );
-
- if ( DEBUG_VARSET )
- var_dump( symbol, value, "set" );
-
- switch ( flag )
- {
- case VAR_SET: /* Replace value */
- list_free( *v );
- *v = value;
- break;
-
- case VAR_APPEND: /* Append value */
- *v = list_append( *v, value );
- break;
-
- case VAR_DEFAULT: /* Set only if unset */
- if ( list_empty( *v ) )
- *v = value;
- else
- list_free( value );
- break;
- }
-}
-
-
-/*
- * var_swap() - swap a variable's value with the given one
- */
-
-LIST * var_swap( struct module_t * module, OBJECT * symbol, LIST * value )
-{
- LIST * * v = var_enter( module, symbol );
- LIST * oldvalue = *v;
- if ( DEBUG_VARSET )
- var_dump( symbol, value, "set" );
- *v = value;
- return oldvalue;
-}
-
-
-/*
- * var_enter() - make new var symbol table entry, returning var ptr
- */
-
-static LIST * * var_enter( struct module_t * module, OBJECT * symbol )
-{
- int found;
- VARIABLE * v;
- int n;
-
- if ( ( n = module_get_fixed_var( module, symbol ) ) != -1 )
- return &module->fixed_variables[ n ];
-
- if ( !module->variables )
- module->variables = hashinit( sizeof( VARIABLE ), "variables" );
-
- v = (VARIABLE *)hash_insert( module->variables, symbol, &found );
- if ( !found )
- {
- v->symbol = object_copy( symbol );
- v->value = L0;
- }
-
- return &v->value;
-}
-
-
-/*
- * var_dump() - dump a variable to stdout
- */
-
-static void var_dump( OBJECT * symbol, LIST * value, char * what )
-{
- printf( "%s %s = ", what, object_str( symbol ) );
- list_print( value );
- printf( "\n" );
-}
-
-
-/*
- * var_done() - free variable tables
- */
-
-static void delete_var_( void * xvar, void * data )
-{
- VARIABLE * const v = (VARIABLE *)xvar;
- object_free( v->symbol );
- list_free( v->value );
-}
-
-void var_done( struct module_t * module )
-{
- list_free( saved_var );
- saved_var = L0;
- hashenumerate( module->variables, delete_var_, 0 );
- hash_free( module->variables );
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/variable.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/variable.h b/ext/kenlm/jam-files/engine/variable.h
deleted file mode 100644
index ddb452b..0000000
--- a/ext/kenlm/jam-files/engine/variable.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Copyright 1993, 2000 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/*
- * variable.h - handle jam multi-element variables
- */
-
-#ifndef VARIABLE_SW20111119_H
-#define VARIABLE_SW20111119_H
-
-#include "lists.h"
-#include "object.h"
-
-
-struct module_t;
-
-void var_defines( struct module_t *, char * const * e, int preprocess );
-LIST * var_get( struct module_t *, OBJECT * symbol );
-void var_set( struct module_t *, OBJECT * symbol, LIST * value, int flag );
-LIST * var_swap( struct module_t *, OBJECT * symbol, LIST * value );
-void var_done( struct module_t * );
-
-/*
- * Defines for var_set().
- */
-
-#define VAR_SET 0 /* override previous value */
-#define VAR_APPEND 1 /* append to previous value */
-#define VAR_DEFAULT 2 /* set only if no previous value */
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/w32_getreg.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/w32_getreg.c b/ext/kenlm/jam-files/engine/w32_getreg.c
deleted file mode 100644
index dd2d0fc..0000000
--- a/ext/kenlm/jam-files/engine/w32_getreg.c
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
-Copyright Paul Lin 2003. Copyright 2006 Bojan Resnik.
-Distributed under the Boost Software License, Version 1.0. (See accompanying
-file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
-*/
-
-# include "jam.h"
-
-# if defined( OS_NT ) || defined( OS_CYGWIN )
-
-# include "lists.h"
-# include "object.h"
-# include "parse.h"
-# include "frames.h"
-# include "strings.h"
-
-# define WIN32_LEAN_AND_MEAN
-# include <windows.h>
-
-# define MAX_REGISTRY_DATA_LENGTH 4096
-# define MAX_REGISTRY_KEYNAME_LENGTH 256
-# define MAX_REGISTRY_VALUENAME_LENGTH 16384
-
-typedef struct
-{
- LPCSTR name;
- HKEY value;
-} KeyMap;
-
-static const KeyMap dlRootKeys[] = {
- { "HKLM", HKEY_LOCAL_MACHINE },
- { "HKCU", HKEY_CURRENT_USER },
- { "HKCR", HKEY_CLASSES_ROOT },
- { "HKEY_LOCAL_MACHINE", HKEY_LOCAL_MACHINE },
- { "HKEY_CURRENT_USER", HKEY_CURRENT_USER },
- { "HKEY_CLASSES_ROOT", HKEY_CLASSES_ROOT },
- { 0, 0 }
-};
-
-static HKEY get_key(char const** path)
-{
- const KeyMap *p;
-
- for (p = dlRootKeys; p->name; ++p)
- {
- int n = strlen(p->name);
- if (!strncmp(*path,p->name,n))
- {
- if ((*path)[n] == '\\' || (*path)[n] == 0)
- {
- *path += n + 1;
- break;
- }
- }
- }
-
- return p->value;
-}
-
-LIST * builtin_system_registry( FRAME * frame, int flags )
-{
- char const* path = object_str( list_front( lol_get(frame->args, 0) ) );
- LIST* result = L0;
- HKEY key = get_key(&path);
-
- if (
- key != 0
- && ERROR_SUCCESS == RegOpenKeyEx(key, path, 0, KEY_QUERY_VALUE, &key)
- )
- {
- DWORD type;
- BYTE data[MAX_REGISTRY_DATA_LENGTH];
- DWORD len = sizeof(data);
- LIST * const field = lol_get(frame->args, 1);
-
- if ( ERROR_SUCCESS ==
- RegQueryValueEx(key, field ? object_str( list_front( field ) ) : 0, 0, &type, data, &len) )
- {
- switch (type)
- {
-
- case REG_EXPAND_SZ:
- {
- long len;
- string expanded[1];
- string_new(expanded);
-
- while (
- (len = ExpandEnvironmentStrings(
- (LPCSTR)data, expanded->value, expanded->capacity))
- > expanded->capacity
- )
- string_reserve(expanded, len);
-
- expanded->size = len - 1;
-
- result = list_push_back( result, object_new(expanded->value) );
- string_free( expanded );
- }
- break;
-
- case REG_MULTI_SZ:
- {
- char* s;
-
- for (s = (char*)data; *s; s += strlen(s) + 1)
- result = list_push_back( result, object_new(s) );
-
- }
- break;
-
- case REG_DWORD:
- {
- char buf[100];
- sprintf( buf, "%u", *(PDWORD)data );
- result = list_push_back( result, object_new(buf) );
- }
- break;
-
- case REG_SZ:
- result = list_push_back( result, object_new( (const char *)data ) );
- break;
- }
- }
- RegCloseKey(key);
- }
- return result;
-}
-
-static LIST* get_subkey_names(HKEY key, char const* path)
-{
- LIST* result = 0;
-
- if ( ERROR_SUCCESS ==
- RegOpenKeyEx(key, path, 0, KEY_ENUMERATE_SUB_KEYS, &key)
- )
- {
- char name[MAX_REGISTRY_KEYNAME_LENGTH];
- DWORD name_size = sizeof(name);
- DWORD index;
- FILETIME last_write_time;
-
- for ( index = 0;
- ERROR_SUCCESS == RegEnumKeyEx(
- key, index, name, &name_size, 0, 0, 0, &last_write_time);
- ++index,
- name_size = sizeof(name)
- )
- {
- name[name_size] = 0;
- result = list_append(result, list_new(object_new(name)));
- }
-
- RegCloseKey(key);
- }
-
- return result;
-}
-
-static LIST* get_value_names(HKEY key, char const* path)
-{
- LIST* result = 0;
-
- if ( ERROR_SUCCESS == RegOpenKeyEx(key, path, 0, KEY_QUERY_VALUE, &key) )
- {
- char name[MAX_REGISTRY_VALUENAME_LENGTH];
- DWORD name_size = sizeof(name);
- DWORD index;
-
- for ( index = 0;
- ERROR_SUCCESS == RegEnumValue(
- key, index, name, &name_size, 0, 0, 0, 0);
- ++index,
- name_size = sizeof(name)
- )
- {
- name[name_size] = 0;
- result = list_append(result, list_new(object_new(name)));
- }
-
- RegCloseKey(key);
- }
-
- return result;
-}
-
-LIST * builtin_system_registry_names( FRAME * frame, int flags )
-{
- char const* path = object_str( list_front( lol_get(frame->args, 0) ) );
- char const* result_type = object_str( list_front( lol_get(frame->args, 1) ) );
-
- HKEY key = get_key(&path);
-
- if ( !strcmp(result_type, "subkeys") )
- return get_subkey_names(key, path);
- if ( !strcmp(result_type, "values") )
- return get_value_names(key, path);
- return 0;
-}
-
-# endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/yyacc.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/yyacc.c b/ext/kenlm/jam-files/engine/yyacc.c
deleted file mode 100644
index b5efc96..0000000
--- a/ext/kenlm/jam-files/engine/yyacc.c
+++ /dev/null
@@ -1,268 +0,0 @@
-/* Copyright 2002 Rene Rivera.
-** Distributed under the Boost Software License, Version 1.0.
-** (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-*/
-
-#include <stdio.h>
-#include <string.h>
-#include <ctype.h>
-#include <stdlib.h>
-
-/*
-# yyacc - yacc wrapper
-#
-# Allows tokens to be written as `literal` and then automatically
-# substituted with #defined tokens.
-#
-# Usage:
-# yyacc file.y filetab.h file.yy
-#
-# inputs:
-# file.yy yacc grammar with ` literals
-#
-# outputs:
-# file.y yacc grammar
-# filetab.h array of string <-> token mappings
-#
-# 3-13-93
-# Documented and p moved in sed command (for some reason,
-# s/x/y/p doesn't work).
-# 10-12-93
-# Take basename as second argument.
-# 12-31-96
-# reversed order of args to be compatible with GenFile rule
-# 11-20-2002
-# Reimplemented as a C program for portability. (Rene Rivera)
-*/
-
-void print_usage();
-char * copy_string(char * s, int l);
-char * tokenize_string(char * s);
-int cmp_literal(const void * a, const void * b);
-
-typedef struct
-{
- char * string;
- char * token;
-} literal;
-
-int main(int argc, char ** argv)
-{
- int result = 0;
- if (argc != 4)
- {
- print_usage();
- result = 1;
- }
- else
- {
- FILE * token_output_f = 0;
- FILE * grammar_output_f = 0;
- FILE * grammar_source_f = 0;
-
- grammar_source_f = fopen(argv[3],"r");
- if (grammar_source_f == 0) { result = 1; }
- if (result == 0)
- {
- literal literals[1024];
- int t = 0;
- char l[2048];
- while (1)
- {
- if (fgets(l,2048,grammar_source_f) != 0)
- {
- char * c = l;
- while (1)
- {
- char * c1 = strchr(c,'`');
- if (c1 != 0)
- {
- char * c2 = strchr(c1+1,'`');
- if (c2 != 0)
- {
- literals[t].string = copy_string(c1+1,c2-c1-1);
- literals[t].token = tokenize_string(literals[t].string);
- t += 1;
- c = c2+1;
- }
- else
- break;
- }
- else
- break;
- }
- }
- else
- {
- break;
- }
- }
- literals[t].string = 0;
- literals[t].token = 0;
- qsort(literals,t,sizeof(literal),cmp_literal);
- {
- int p = 1;
- int i = 1;
- while (literals[i].string != 0)
- {
- if (strcmp(literals[p-1].string,literals[i].string) != 0)
- {
- literals[p] = literals[i];
- p += 1;
- }
- i += 1;
- }
- literals[p].string = 0;
- literals[p].token = 0;
- t = p;
- }
- token_output_f = fopen(argv[2],"w");
- if (token_output_f != 0)
- {
- int i = 0;
- while (literals[i].string != 0)
- {
- fprintf(token_output_f," { \"%s\", %s },\n",literals[i].string,literals[i].token);
- i += 1;
- }
- fclose(token_output_f);
- }
- else
- result = 1;
- if (result == 0)
- {
- grammar_output_f = fopen(argv[1],"w");
- if (grammar_output_f != 0)
- {
- int i = 0;
- while (literals[i].string != 0)
- {
- fprintf(grammar_output_f,"%%token %s\n",literals[i].token);
- i += 1;
- }
- rewind(grammar_source_f);
- while (1)
- {
- if (fgets(l,2048,grammar_source_f) != 0)
- {
- char * c = l;
- while (1)
- {
- char * c1 = strchr(c,'`');
- if (c1 != 0)
- {
- char * c2 = strchr(c1+1,'`');
- if (c2 != 0)
- {
- literal key;
- literal * replacement = 0;
- key.string = copy_string(c1+1,c2-c1-1);
- key.token = 0;
- replacement = (literal*)bsearch(
- &key,literals,t,sizeof(literal),cmp_literal);
- *c1 = 0;
- fprintf(grammar_output_f,"%s%s",c,replacement->token);
- c = c2+1;
- }
- else
- {
- fprintf(grammar_output_f,"%s",c);
- break;
- }
- }
- else
- {
- fprintf(grammar_output_f,"%s",c);
- break;
- }
- }
- }
- else
- {
- break;
- }
- }
- fclose(grammar_output_f);
- }
- else
- result = 1;
- }
- }
- if (result != 0)
- {
- perror("yyacc");
- }
- }
- return result;
-}
-
-static char * usage[] = {
- "yyacc <grammar output.y> <token table output.h> <grammar source.yy>",
- 0 };
-
-void print_usage()
-{
- char ** u;
- for (u = usage; *u != 0; ++u)
- {
- fputs(*u,stderr); putc('\n',stderr);
- }
-}
-
-char * copy_string(char * s, int l)
-{
- char * result = (char*)malloc(l+1);
- strncpy(result,s,l);
- result[l] = 0;
- return result;
-}
-
-char * tokenize_string(char * s)
-{
- char * result;
- char * literal = s;
- int l;
- int c;
-
- if (strcmp(s,":") == 0) literal = "_colon";
- else if (strcmp(s,"!") == 0) literal = "_bang";
- else if (strcmp(s,"!=") == 0) literal = "_bang_equals";
- else if (strcmp(s,"&&") == 0) literal = "_amperamper";
- else if (strcmp(s,"&") == 0) literal = "_amper";
- else if (strcmp(s,"+") == 0) literal = "_plus";
- else if (strcmp(s,"+=") == 0) literal = "_plus_equals";
- else if (strcmp(s,"||") == 0) literal = "_barbar";
- else if (strcmp(s,"|") == 0) literal = "_bar";
- else if (strcmp(s,";") == 0) literal = "_semic";
- else if (strcmp(s,"-") == 0) literal = "_minus";
- else if (strcmp(s,"<") == 0) literal = "_langle";
- else if (strcmp(s,"<=") == 0) literal = "_langle_equals";
- else if (strcmp(s,">") == 0) literal = "_rangle";
- else if (strcmp(s,">=") == 0) literal = "_rangle_equals";
- else if (strcmp(s,".") == 0) literal = "_period";
- else if (strcmp(s,"?") == 0) literal = "_question";
- else if (strcmp(s,"?=") == 0) literal = "_question_equals";
- else if (strcmp(s,"=") == 0) literal = "_equals";
- else if (strcmp(s,",") == 0) literal = "_comma";
- else if (strcmp(s,"[") == 0) literal = "_lbracket";
- else if (strcmp(s,"]") == 0) literal = "_rbracket";
- else if (strcmp(s,"{") == 0) literal = "_lbrace";
- else if (strcmp(s,"}") == 0) literal = "_rbrace";
- else if (strcmp(s,"(") == 0) literal = "_lparen";
- else if (strcmp(s,")") == 0) literal = "_rparen";
- l = strlen(literal)+2;
- result = (char*)malloc(l+1);
- for (c = 0; literal[c] != 0; ++c)
- {
- result[c] = toupper(literal[c]);
- }
- result[l-2] = '_';
- result[l-1] = 't';
- result[l] = 0;
- return result;
-}
-
-int cmp_literal(const void * a, const void * b)
-{
- return strcmp(((const literal *)a)->string,((const literal *)b)->string);
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/fail/Jamroot
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/fail/Jamroot b/ext/kenlm/jam-files/fail/Jamroot
deleted file mode 100644
index c3584d8..0000000
--- a/ext/kenlm/jam-files/fail/Jamroot
+++ /dev/null
@@ -1,4 +0,0 @@
-actions fail {
- false
-}
-make fail : : fail ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/sanity.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/sanity.jam b/ext/kenlm/jam-files/sanity.jam
deleted file mode 100644
index 1851ece..0000000
--- a/ext/kenlm/jam-files/sanity.jam
+++ /dev/null
@@ -1,344 +0,0 @@
-import modules ;
-import option ;
-import os ;
-import path ;
-import project ;
-import build-system ;
-import version ;
-
-#Shell with trailing line removed http://lists.boost.org/boost-build/2007/08/17051.php
-rule trim-nl ( str extras * ) {
-return [ MATCH "([^
-]*)" : $(str) ] $(extras) ;
-}
-rule _shell ( cmd : extras * ) {
- return [ trim-nl [ SHELL $(cmd) : $(extras) ] ] ;
-}
-
-rule shell_or_fail ( cmd ) {
- local ret = [ SHELL $(cmd) : exit-status ] ;
- if $(ret[2]) != 0 {
- exit $(cmd) failed : 1 ;
- }
-}
-
-rule shell_or_die ( cmd ) {
- local ret = [ SHELL $(cmd) : exit-status ] ;
- if $(ret[2]) != 0 {
- exit $(cmd) failed : 1 ;
- }
- return [ trim-nl $(ret[1]) ] ;
-}
-
-cxxflags = [ os.environ "CXXFLAGS" ] ;
-cflags = [ os.environ "CFLAGS" ] ;
-ldflags = [ os.environ "LDFLAGS" ] ;
-
-#Run g++ with empty main and these arguments to see if it passes.
-rule test_flags ( flags * : main ? ) {
- flags = $(cxxflags) $(ldflags) $(flags) ;
- if ! $(main) {
- main = "int main() {}" ;
- }
- local cmd = "bash -c \"g++ "$(flags:J=" ")" -x c++ - <<<'$(main)' -o $(TOP)/dummy >/dev/null 2>/dev/null && rm $(TOP)/dummy 2>/dev/null\"" ;
- local ret = [ SHELL $(cmd) : exit-status ] ;
- if --debug-configuration in [ modules.peek : ARGV ] {
- echo $(cmd) ;
- echo $(ret) ;
- }
- if $(ret[2]) = 0 {
- return true ;
- } else {
- return ;
- }
-}
-
-rule test_header ( name ) {
- return [ test_flags "-include $(name)" ] ;
-}
-
-requirements = ;
-
-FORCE-STATIC = [ option.get "static" : : "yes" ] ;
-if $(FORCE-STATIC) {
- requirements += <link>static <runtime-link>static ;
-}
-
-rule test_library ( name ) {
- if $(FORCE-STATIC) {
- return [ test_flags "-Wl,-Bstatic -l$(name) -Wl,-Bdynamic" ] ;
- } else {
- return [ test_flags "-l$(name)" ] ;
- }
-}
-
-{
- local cleaning = [ option.get "clean" : : yes ] ;
- cleaning ?= [ option.get "clean-all" : no : yes ] ;
- if "clean" in [ modules.peek : ARGV ] {
- cleaning = yes ;
- }
- constant CLEANING : $(cleaning) ;
-}
-
-shared-command-line = ;
-local argv = [ modules.peek : ARGV ] ;
-while $(argv) {
- if $(argv[1]) = "link=shared" {
- shared-command-line = <link>shared ;
- }
- argv = $(argv[2-]) ;
-}
-
-#Determine if a library can be compiled statically.
-rule auto-shared ( name : additional * ) {
-
- additional ?= "" ;
- if $(shared-command-line) = "<link>shared" {
- return "<link>shared" ;
- } else {
- if [ test_flags $(additional)" -Wl,-Bstatic -l"$(name)" -Wl,-Bdynamic" ] {
- return ;
- } else {
- if $(FORCE-STATIC) {
- echo "Could not statically link against lib $(name). Your build will probably fail." ;
- return ;
- } else {
- return "<link>shared" ;
- }
- }
- }
-}
-
-# MacPorts' default location is /opt/local -- use this if no path is given.
-with-macports = [ option.get "with-macports" : : "/opt/local" ] ;
-if $(with-macports) {
- using darwin ;
- ECHO "Using --with-macports=$(with-macports), implying use of darwin GCC" ;
-
- L-boost-search = -L$(with-macports)/lib ;
- boost-search = <search>$(with-macports)/lib ;
- I-boost-include = -I$(with-macports)/include ;
- boost-include = <include>$(with-macports)/include ;
- requirements += $(boost-include) ;
-} else {
- with-boost = [ option.get "with-boost" ] ;
- with-boost ?= [ os.environ "BOOST_ROOT" ] ;
- if $(with-boost) {
- L-boost-search = -L$(with-boost)/lib" "-L$(with-boost)/lib64 ;
- boost-search = <search>$(with-boost)/lib <search>$(with-boost)/lib64 ;
- I-boost-include = -I$(with-boost)/include ;
- boost-include = <include>$(with-boost)/include ;
- requirements += $(boost-include) ;
- } else {
- L-boost-search = "" ;
- boost-search = ;
- I-boost-include = "" ;
- boost-include = ;
- }
-}
-
-#Convenience rule for boost libraries. Defines library boost_$(name).
-rule boost-lib ( name macro : deps * ) {
- lib boost_$(name)_static : $(deps) : $(boost-search) <name>boost_$(name)$(boost-lib-version) <link>static ;
- lib boost_$(name)_shared : $(deps) : $(boost-search) <name>boost_$(name)$(boost-lib-version) <link>shared : : <define>BOOST_$(macro) ;
-
- alias boost_$(name)_default : $(deps) : <link>static:<source>boost_$(name)_static <link>shared:<source>boost_$(name)_shared ;
-
- alias boost_$(name)_static_works : $(deps) : [ check-target-builds empty_test_shared "Shared Boost" : <source>boost_$(name)_default : <source>boost_$(name)_static ] ;
- alias boost_$(name) : $(deps) : [ check-target-builds empty_test_static "Static Boost" : <source>boost_$(name)_static_works : <source>boost_$(name)_shared ] ;
-}
-
-#Argument is e.g. 103600
-rule boost ( min-version ) {
- local cmd = "bash -c \"g++ "$(I-boost-include)" -dM -x c++ -E /dev/null -include boost/version.hpp 2>/dev/null |grep '#define BOOST_'\"" ;
- local boost-shell = [ SHELL "$(cmd)" : exit-status ] ;
- if $(boost-shell[2]) != 0 && $(CLEANING) = no {
- echo Failed to run "$(cmd)" ;
- exit Boost does not seem to be installed or g++ is confused. : 1 ;
- }
- constant BOOST-VERSION : [ MATCH "#define BOOST_VERSION ([0-9]*)" : $(boost-shell[1]) ] ;
- if $(BOOST-VERSION) < $(min-version) && $(CLEANING) = no {
- exit You have Boost $(BOOST-VERSION). This package requires Boost at least $(min-version) (and preferably newer). : 1 ;
- }
- # If matching version tags exist, use them.
- boost-lib-version = [ MATCH "#define BOOST_LIB_VERSION \"([^\"]*)\"" : $(boost-shell[1]) ] ;
- if [ test_flags $(L-boost-search)" -lboost_program_options-"$(boost-lib-version) ] {
- boost-lib-version = "-"$(boost-lib-version) ;
- } else {
- boost-lib-version = "" ;
- }
-
- #Crazy amount of testing to make sure that BOOST_TEST_DYN_LINK is defined properly.
- lib boost_unit_test_framework_static_test : : $(boost-search) <name>boost_unit_test_framework$(boost-lib-version) <link>static ;
- obj empty_test_static.o : jam-files/empty_test_main.cc boost_unit_test_framework_static_test : $(boost-include) ;
- exe empty_test_static : empty_test_static.o boost_unit_test_framework_static_test ;
-
- lib boost_unit_test_framework_shared_test : : $(boost-search) <name>boost_unit_test_framework$(boost-lib-version) <link>shared : : <define>BOOST_TEST_DYN_LINK ;
- obj empty_test_shared.o : jam-files/empty_test_main.cc boost_unit_test_framework_shared_test : $(boost-include) ;
- exe empty_test_shared : empty_test_shared.o boost_unit_test_framework_shared_test ;
-
- explicit empty_test_static.o empty_test_static empty_test_shared.o empty_test_shared ;
-
-
- #See tools/build/v2/contrib/boost.jam in a boost distribution for a table of macros to define.
- boost-lib system SYSTEM_DYN_LINK ;
- boost-lib thread THREAD_DYN_DLL : boost_system ;
- boost-lib program_options PROGRAM_OPTIONS_DYN_LINK ;
- boost-lib iostreams IOSTREAMS_DYN_LINK ;
- boost-lib filesystem FILE_SYSTEM_DYN_LINK ;
- boost-lib unit_test_framework TEST_DYN_LINK ;
-# if $(BOOST-VERSION) >= 104800 {
-# boost-lib chrono CHRONO_DYN_LINK ;
-# boost-lib timer TIMER_DYN_LINK : boost_chrono ;
-# }
-}
-
-#Link normally to a library, but sometimes static isn't installed so fall back to dynamic.
-rule external-lib ( name : search-path * : deps * ) {
- lib $(name) : : [ auto-shared $(name) : "-L"$(search-path) ] <search>$(search-path) <use>$(deps) ;
-}
-
-#Write the current command line to previous.sh. This does not do shell escaping.
-{
- local build-log = $(TOP)/previous.sh ;
- if ! [ path.exists $(build-log) ] {
- SHELL "touch \"$(build-log)\" && chmod +x \"$(build-log)\"" ;
- }
- local script = [ modules.peek : ARGV ] ;
- if $(script[1]) = "./jam-files/bjam" {
- #The ./bjam shell script calls ./jam-files/bjam so that appears in argv but
- #we want ./bjam to appear so the environment variables are set correctly.
- script = "./bjam "$(script[2-]:J=" ") ;
- } else {
- script = $(script:J=" ") ;
- }
- script = "#!/bin/sh\n$(script)\n" ;
- local ignored = @($(build-log):E=$(script)) ;
-}
-
-#Boost jam's static clang for Linux is buggy.
-requirements += <cxxflags>$(cxxflags) <cflags>$(cflags) <linkflags>$(ldflags) <os>LINUX,<toolset>clang:<link>shared ;
-
-if ! [ option.get "without-libsegfault" : : "yes" ] && ! $(FORCE-STATIC) {
- #libSegFault prints a stack trace on segfault. Link against it if available.
- if [ test_flags "-lSegFault" ] {
- external-lib SegFault ;
- requirements += <library>SegFault ;
- }
-}
-
-if [ option.get "git" : : "yes" ] {
- local revision = [ _shell "git rev-parse --verify HEAD |head -c 7" ] ;
- constant GITTAG : "/"$(revision) ;
-} else {
- constant GITTAG : "" ;
-}
-
-local prefix = [ option.get "prefix" ] ;
-if $(prefix) {
- prefix = [ path.root $(prefix) [ path.pwd ] ] ;
- prefix = $(prefix)$(GITTAG) ;
-} else {
- prefix = $(TOP)$(GITTAG) ;
-}
-
-path-constant PREFIX : $(prefix) ;
-
-path-constant BINDIR : [ option.get "bindir" : $(PREFIX)/bin ] ;
-path-constant LIBDIR : [ option.get "libdir" : $(PREFIX)/lib ] ;
-rule install-bin-libs ( deps * ) {
- install prefix-bin : $(deps) : <location>$(BINDIR) <install-dependencies>on <install-type>EXE <link>shared:<dll-path>$(LIBDIR) ;
- install prefix-lib : $(deps) : <location>$(LIBDIR) <install-dependencies>on <install-type>LIB <link>shared:<dll-path>$(LIBDIR) ;
-}
-rule install-headers ( name : list * : source-root ? ) {
- local includedir = [ option.get "includedir" : $(prefix)/include ] ;
- source-root ?= "." ;
- install $(name) : $(list) : <location>$(includedir) <install-source-root>$(source-root) ;
-}
-
-rule build-projects ( projects * ) {
- for local p in $(projects) {
- build-project $(p) ;
- }
-}
-
-#Only one post build hook is allowed. Allow multiple.
-post-hooks = ;
-rule post-build ( ok ? ) {
- for local r in $(post-hooks) {
- $(r) $(ok) ;
- }
-}
-IMPORT $(__name__) : post-build : : $(__name__).post-build ;
-build-system.set-post-build-hook $(__name__).post-build ;
-rule add-post-hook ( names * ) {
- post-hooks += $(names) ;
-}
-
-rule failure-message ( ok ? ) {
- if $(ok) != "ok" {
- local args = [ modules.peek : ARGV ] ;
- local args = $(args:J=" ") ;
- if --debug-configuration in [ modules.peek : ARGV ] {
- echo "The build failed with command line: " ;
- echo " $(args)" ;
- echo "If you need support, attach the full output to your e-mail." ;
- } else {
- echo "The build failed. If you need support, run:" ;
- echo " $(args) --debug-configuration -d2 |gzip >build.log.gz" ;
- echo "then attach build.log.gz to your e-mail." ;
- }
- echo "ERROR" ;
- } else {
- echo "SUCCESS" ;
- }
-}
-add-post-hook failure-message ;
-
-import feature : feature ;
-feature options-to-write : : free ;
-import toolset : flags ;
-flags write-options OPTIONS-TO-WRITE <options-to-write> ;
-actions write-options {
- echo "$(OPTIONS-TO-WRITE)" > $(<) ;
-}
-
-#Compare contents of file with current. If they're different, write to the
-#file. This file can then be used with <dependency>$(file) to force
-#recompilation.
-rule update-if-changed ( file current ) {
- if ( ! [ path.exists $(file) ] ) || ( [ _shell "cat $(file)" ] != $(current) ) {
- make $(file) : : $(__name__).write-options : <options-to-write>$(current) ;
- always $(file) ;
- }
-}
-
-if [ option.get "sanity-test" : : "yes" ] {
- local current_version = [ modules.peek : JAM_VERSION ] ;
- if ( $(current_version[0]) < 2000 && [ version.check-jam-version 3 1 16 ] ) || [ version.check-jam-version 2011 0 0 ] {
- EXIT "Sane" : 0 ;
- } else {
- EXIT "Bad" : 1 ;
- }
-}
-
-#Hack to act like alias in the sense that no lib is built, but only build cpp files once.
-import type ;
-rule fakelib ( name : deps * : requirements * : default-build * : usage-requirements * ) {
- local c-files = ;
- local real-deps = ;
- for local c in $(deps) {
- if [ type.type $(c) ] = CPP {
- c-files += $(c) ;
- } else {
- real-deps += $(c) ;
- }
- }
- for local c in $(c-files) {
- obj $(c:B).o : $(c) $(real-deps) : $(requirements) : $(default-build) : $(usage_requirements) ;
- }
- alias $(name) : $(c-files:B).o $(real-deps) : $(requirements) : $(default-build) : $(usage-requirements) ;
-}
-
-use-project /top : . ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/CMakeLists.txt b/ext/kenlm/lm/CMakeLists.txt
deleted file mode 100644
index e3ef06f..0000000
--- a/ext/kenlm/lm/CMakeLists.txt
+++ /dev/null
@@ -1,90 +0,0 @@
-cmake_minimum_required(VERSION 2.8.8)
-#
-# The KenLM cmake files make use of add_library(... OBJECTS ...)
-#
-# This syntax allows grouping of source files when compiling
-# (effectively creating "fake" libraries based on source subdirs).
-#
-# This syntax was only added in cmake version 2.8.8
-#
-# see http://www.cmake.org/Wiki/CMake/Tutorials/Object_Library
-
-
-# This CMake file was created by Lane Schwartz <do...@gmail.com>
-
-
-set(KENLM_MAX_ORDER 6 CACHE STRING "Maximum supported ngram order")
-
-add_definitions(-DKENLM_MAX_ORDER=${KENLM_MAX_ORDER})
-
-
-# Explicitly list the source files for this subdirectory
-#
-# If you add any source files to this subdirectory
-# that should be included in the kenlm library,
-# (this excludes any unit test files)
-# you should add them to the following list:
-set(KENLM_SOURCE
- bhiksha.cc
- binary_format.cc
- config.cc
- lm_exception.cc
- model.cc
- quantize.cc
- read_arpa.cc
- search_hashed.cc
- search_trie.cc
- sizes.cc
- trie.cc
- trie_sort.cc
- value_build.cc
- virtual_interface.cc
- vocab.cc
-)
-
-
-# Group these objects together for later use.
-#
-# Given add_library(foo OBJECT ${my_foo_sources}),
-# refer to these objects as $<TARGET_OBJECTS:foo>
-#
-add_library(kenlm OBJECT ${KENLM_SOURCE})
-
-# This directory has children that need to be processed
-add_subdirectory(builder)
-add_subdirectory(common)
-add_subdirectory(filter)
-
-
-
-# Explicitly list the executable files to be compiled
-set(EXE_LIST
- query
- fragment
- build_binary
-)
-
-AddExes(EXES ${EXE_LIST}
- DEPENDS $<TARGET_OBJECTS:kenlm> $<TARGET_OBJECTS:kenlm_util>
- LIBRARIES ${Boost_LIBRARIES} pthread)
-
-# Conditionally build the interpolation code
-if(BUILD_INTERPOLATE)
- add_subdirectory(interpolate)
-endif()
-
-if(BUILD_TESTING)
-
- set(KENLM_BOOST_TESTS_LIST left_test partial_test)
- AddTests(TESTS ${KENLM_BOOST_TESTS_LIST}
- DEPENDS $<TARGET_OBJECTS:kenlm> $<TARGET_OBJECTS:kenlm_util>
- LIBRARIES ${Boost_LIBRARIES} pthread
- TEST_ARGS ${CMAKE_CURRENT_SOURCE_DIR}/test.arpa)
-
- # model_test requires an extra command line parameter
- KenLMAddTest(TEST model_test
- DEPENDS $<TARGET_OBJECTS:kenlm> $<TARGET_OBJECTS:kenlm_util>
- LIBRARIES ${Boost_LIBRARIES} pthread
- TEST_ARGS ${CMAKE_CURRENT_SOURCE_DIR}/test.arpa
- ${CMAKE_CURRENT_SOURCE_DIR}/test_nounk.arpa)
-endif()
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/Jamfile
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/Jamfile b/ext/kenlm/lm/Jamfile
deleted file mode 100644
index a479e2d..0000000
--- a/ext/kenlm/lm/Jamfile
+++ /dev/null
@@ -1,40 +0,0 @@
-# If you need higher order, change this option
-# Having this limit means that State can be
-# (KENLM_MAX_ORDER - 1) * sizeof(float) bytes instead of
-# sizeof(float*) + (KENLM_MAX_ORDER - 1) * sizeof(float) + malloc overhead
-max-order = [ option.get "max-kenlm-order" : 6 : 6 ] ;
-if ( $(max-order) != 6 ) {
- echo "Setting KenLM maximum n-gram order to $(max-order)" ;
-}
-max-order = <define>KENLM_MAX_ORDER=$(max-order) ;
-
-path-constant ORDER-LOG : bin/order.log ;
-update-if-changed $(ORDER-LOG) $(max-order) ;
-
-max-order += <dependency>$(ORDER-LOG) ;
-
-wrappers = ;
-local with-nplm = [ option.get "with-nplm" ] ;
-if $(with-nplm) {
- lib nplm : : <search>$(with-nplm)/src ;
- obj nplm.o : wrappers/nplm.cc : <include>.. <include>$(with-nplm)/src <cxxflags>-fopenmp <include>$(with-nplm)/3rdparty/eigen <define>NPLM_DOUBLE_PRECISION=0 ;
- alias nplm-all : nplm.o nplm ..//boost_thread : : : <cxxflags>-fopenmp <linkflags>-fopenmp <define>WITH_NPLM <library>..//boost_thread ;
- wrappers += nplm-all ;
-}
-
-fakelib kenlm : $(wrappers) [ glob *.cc : *main.cc *test.cc ] ../util//kenutil : <include>.. $(max-order) : : <include>.. $(max-order) ;
-
-import testing ;
-
-run left_test.cc kenlm /top//boost_unit_test_framework : : test.arpa ;
-run model_test.cc kenlm /top//boost_unit_test_framework : : test.arpa test_nounk.arpa ;
-run partial_test.cc kenlm /top//boost_unit_test_framework : : test.arpa ;
-
-exes = ;
-for local p in [ glob *_main.cc ] {
- local name = [ MATCH "(.*)\_main.cc" : $(p) ] ;
- exe $(name) : $(p) kenlm ;
- exes += $(name) ;
-}
-
-alias programs : $(exes) filter//filter builder//dump_counts : <threading>multi:<source>builder//lmplz ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/bhiksha.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/bhiksha.cc b/ext/kenlm/lm/bhiksha.cc
deleted file mode 100644
index 4262b61..0000000
--- a/ext/kenlm/lm/bhiksha.cc
+++ /dev/null
@@ -1,94 +0,0 @@
-#include "lm/bhiksha.hh"
-
-#include "lm/binary_format.hh"
-#include "lm/config.hh"
-#include "util/file.hh"
-#include "util/exception.hh"
-
-#include <limits>
-
-namespace lm {
-namespace ngram {
-namespace trie {
-
-DontBhiksha::DontBhiksha(const void * /*base*/, uint64_t /*max_offset*/, uint64_t max_next, const Config &/*config*/) :
- next_(util::BitsMask::ByMax(max_next)) {}
-
-const uint8_t kArrayBhikshaVersion = 0;
-
-// TODO: put this in binary file header instead when I change the binary file format again.
-void ArrayBhiksha::UpdateConfigFromBinary(const BinaryFormat &file, uint64_t offset, Config &config) {
- uint8_t buffer[2];
- file.ReadForConfig(buffer, 2, offset);
- uint8_t version = buffer[0];
- uint8_t configured_bits = buffer[1];
- if (version != kArrayBhikshaVersion) UTIL_THROW(FormatLoadException, "This file has sorted array compression version " << (unsigned) version << " but the code expects version " << (unsigned)kArrayBhikshaVersion);
- config.pointer_bhiksha_bits = configured_bits;
-}
-
-namespace {
-
-// Find argmin_{chopped \in [0, RequiredBits(max_next)]} ChoppedDelta(max_offset)
-uint8_t ChopBits(uint64_t max_offset, uint64_t max_next, const Config &config) {
- uint8_t required = util::RequiredBits(max_next);
- uint8_t best_chop = 0;
- int64_t lowest_change = std::numeric_limits<int64_t>::max();
- // There are probably faster ways but I don't care because this is only done once per order at construction time.
- for (uint8_t chop = 0; chop <= std::min(required, config.pointer_bhiksha_bits); ++chop) {
- int64_t change = (max_next >> (required - chop)) * 64 /* table cost in bits */
- - max_offset * static_cast<int64_t>(chop); /* savings in bits*/
- if (change < lowest_change) {
- lowest_change = change;
- best_chop = chop;
- }
- }
- return best_chop;
-}
-
-std::size_t ArrayCount(uint64_t max_offset, uint64_t max_next, const Config &config) {
- uint8_t required = util::RequiredBits(max_next);
- uint8_t chopping = ChopBits(max_offset, max_next, config);
- return (max_next >> (required - chopping)) + 1 /* we store 0 too */;
-}
-} // namespace
-
-uint64_t ArrayBhiksha::Size(uint64_t max_offset, uint64_t max_next, const Config &config) {
- return sizeof(uint64_t) * (1 /* header */ + ArrayCount(max_offset, max_next, config)) + 7 /* 8-byte alignment */;
-}
-
-uint8_t ArrayBhiksha::InlineBits(uint64_t max_offset, uint64_t max_next, const Config &config) {
- return util::RequiredBits(max_next) - ChopBits(max_offset, max_next, config);
-}
-
-namespace {
-
-void *AlignTo8(void *from) {
- uint8_t *val = reinterpret_cast<uint8_t*>(from);
- std::size_t remainder = reinterpret_cast<std::size_t>(val) & 7;
- if (!remainder) return val;
- return val + 8 - remainder;
-}
-
-} // namespace
-
-ArrayBhiksha::ArrayBhiksha(void *base, uint64_t max_offset, uint64_t max_next, const Config &config)
- : next_inline_(util::BitsMask::ByBits(InlineBits(max_offset, max_next, config))),
- offset_begin_(reinterpret_cast<const uint64_t*>(AlignTo8(base)) + 1 /* 8-byte header */),
- offset_end_(offset_begin_ + ArrayCount(max_offset, max_next, config)),
- write_to_(reinterpret_cast<uint64_t*>(AlignTo8(base)) + 1 /* 8-byte header */ + 1 /* first entry is 0 */),
- original_base_(base) {}
-
-void ArrayBhiksha::FinishedLoading(const Config &config) {
- // *offset_begin_ = 0 but without a const_cast.
- *(write_to_ - (write_to_ - offset_begin_)) = 0;
-
- if (write_to_ != offset_end_) UTIL_THROW(util::Exception, "Did not get all the array entries that were expected.");
-
- uint8_t *head_write = reinterpret_cast<uint8_t*>(original_base_);
- *(head_write++) = kArrayBhikshaVersion;
- *(head_write++) = config.pointer_bhiksha_bits;
-}
-
-} // namespace trie
-} // namespace ngram
-} // namespace lm
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/bhiksha.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/bhiksha.hh b/ext/kenlm/lm/bhiksha.hh
deleted file mode 100644
index 36438f1..0000000
--- a/ext/kenlm/lm/bhiksha.hh
+++ /dev/null
@@ -1,122 +0,0 @@
-/* Simple implementation of
- * @inproceedings{bhikshacompression,
- * author={Bhiksha Raj and Ed Whittaker},
- * year={2003},
- * title={Lossless Compression of Language Model Structure and Word Identifiers},
- * booktitle={Proceedings of IEEE International Conference on Acoustics, Speech and Signal Processing},
- * pages={388--391},
- * }
- *
- * Currently only used for next pointers.
- */
-
-#ifndef LM_BHIKSHA_H
-#define LM_BHIKSHA_H
-
-#include "lm/model_type.hh"
-#include "lm/trie.hh"
-#include "util/bit_packing.hh"
-#include "util/sorted_uniform.hh"
-
-#include <algorithm>
-#include <stdint.h>
-#include <cassert>
-
-namespace lm {
-namespace ngram {
-struct Config;
-class BinaryFormat;
-
-namespace trie {
-
-class DontBhiksha {
- public:
- static const ModelType kModelTypeAdd = static_cast<ModelType>(0);
-
- static void UpdateConfigFromBinary(const BinaryFormat &, uint64_t, Config &/*config*/) {}
-
- static uint64_t Size(uint64_t /*max_offset*/, uint64_t /*max_next*/, const Config &/*config*/) { return 0; }
-
- static uint8_t InlineBits(uint64_t /*max_offset*/, uint64_t max_next, const Config &/*config*/) {
- return util::RequiredBits(max_next);
- }
-
- DontBhiksha(const void *base, uint64_t max_offset, uint64_t max_next, const Config &config);
-
- void ReadNext(const void *base, uint64_t bit_offset, uint64_t /*index*/, uint8_t total_bits, NodeRange &out) const {
- out.begin = util::ReadInt57(base, bit_offset, next_.bits, next_.mask);
- out.end = util::ReadInt57(base, bit_offset + total_bits, next_.bits, next_.mask);
- //assert(out.end >= out.begin);
- }
-
- void WriteNext(void *base, uint64_t bit_offset, uint64_t /*index*/, uint64_t value) {
- util::WriteInt57(base, bit_offset, next_.bits, value);
- }
-
- void FinishedLoading(const Config &/*config*/) {}
-
- uint8_t InlineBits() const { return next_.bits; }
-
- private:
- util::BitsMask next_;
-};
-
-class ArrayBhiksha {
- public:
- static const ModelType kModelTypeAdd = kArrayAdd;
-
- static void UpdateConfigFromBinary(const BinaryFormat &file, uint64_t offset, Config &config);
-
- static uint64_t Size(uint64_t max_offset, uint64_t max_next, const Config &config);
-
- static uint8_t InlineBits(uint64_t max_offset, uint64_t max_next, const Config &config);
-
- ArrayBhiksha(void *base, uint64_t max_offset, uint64_t max_value, const Config &config);
-
- void ReadNext(const void *base, uint64_t bit_offset, uint64_t index, uint8_t total_bits, NodeRange &out) const {
- // Some assertions are commented out because they are expensive.
- // assert(*offset_begin_ == 0);
- // std::upper_bound returns the first element that is greater. Want the
- // last element that is <= to the index.
- const uint64_t *begin_it = std::upper_bound(offset_begin_, offset_end_, index) - 1;
- // Since *offset_begin_ == 0, the position should be in range.
- // assert(begin_it >= offset_begin_);
- const uint64_t *end_it;
- for (end_it = begin_it + 1; (end_it < offset_end_) && (*end_it <= index + 1); ++end_it) {}
- // assert(end_it == std::upper_bound(offset_begin_, offset_end_, index + 1));
- --end_it;
- // assert(end_it >= begin_it);
- out.begin = ((begin_it - offset_begin_) << next_inline_.bits) |
- util::ReadInt57(base, bit_offset, next_inline_.bits, next_inline_.mask);
- out.end = ((end_it - offset_begin_) << next_inline_.bits) |
- util::ReadInt57(base, bit_offset + total_bits, next_inline_.bits, next_inline_.mask);
- // If this fails, consider rebuilding your model using KenLM after 1e333d786b748555e8f368d2bbba29a016c98052
- assert(out.end >= out.begin);
- }
-
- void WriteNext(void *base, uint64_t bit_offset, uint64_t index, uint64_t value) {
- uint64_t encode = value >> next_inline_.bits;
- for (; write_to_ <= offset_begin_ + encode; ++write_to_) *write_to_ = index;
- util::WriteInt57(base, bit_offset, next_inline_.bits, value & next_inline_.mask);
- }
-
- void FinishedLoading(const Config &config);
-
- uint8_t InlineBits() const { return next_inline_.bits; }
-
- private:
- const util::BitsMask next_inline_;
-
- const uint64_t *const offset_begin_;
- const uint64_t *const offset_end_;
-
- uint64_t *write_to_;
-
- void *original_base_;
-};
-
-} // namespace trie
-} // namespace ngram
-} // namespace lm
-
-#endif // LM_BHIKSHA_H
[41/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/builtin.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/builtin.jam b/ext/kenlm/jam-files/boost-build/tools/builtin.jam
deleted file mode 100644
index 01c82f1..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/builtin.jam
+++ /dev/null
@@ -1,974 +0,0 @@
-# Copyright 2002, 2003, 2004, 2005 Dave Abrahams
-# Copyright 2002, 2005, 2006, 2007, 2010 Rene Rivera
-# Copyright 2006 Juergen Hunold
-# Copyright 2005 Toon Knapen
-# Copyright 2002, 2003, 2004, 2005, 2006 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# Defines standard features and rules.
-
-import alias ;
-import "class" : new ;
-import errors ;
-import feature ;
-import generators ;
-import numbers ;
-import os ;
-import path ;
-import print ;
-import project ;
-import property ;
-import regex ;
-import scanner ;
-import sequence ;
-import stage ;
-import symlink ;
-import toolset ;
-import type ;
-import targets ;
-import types/register ;
-import utility ;
-import virtual-target ;
-import message ;
-import convert ;
-
-# FIXME: the following generate module import is not needed here but removing it
-# too hastly will break using code (e.g. the main Boost library Jamroot file)
-# that forgot to import the generate module before calling the generate rule.
-import generate ;
-
-
-.os-names = aix bsd cygwin darwin freebsd hpux iphone linux netbsd openbsd osf
- qnx qnxnto sgi solaris unix unixware windows
- elf # Not actually an OS -- used for targeting bare metal where object
- # format is ELF. This catches both -elf and -eabi gcc targets and well
- # as other compilers targeting ELF. It is not clear how often we need
- # the 'elf' key as opposed to other bare metal targets, but let us
- # stick with gcc naming.
- ;
-
-# Feature used to determine which OS we're on. New <target-os> and <host-os>
-# features should be used instead.
-local os = [ modules.peek : OS ] ;
-feature.feature os : $(os) : propagated link-incompatible ;
-
-
-# Translates from bjam current OS to the os tags used in host-os and target-os,
-# i.e. returns the running host-os.
-#
-local rule default-host-os ( )
-{
- local host-os ;
- if [ os.name ] in $(.os-names:U)
- {
- host-os = [ os.name ] ;
- }
- else
- {
- switch [ os.name ]
- {
- case NT : host-os = windows ;
- case AS400 : host-os = unix ;
- case MINGW : host-os = windows ;
- case BSDI : host-os = bsd ;
- case COHERENT : host-os = unix ;
- case DRAGONFLYBSD : host-os = bsd ;
- case IRIX : host-os = sgi ;
- case MACOSX : host-os = darwin ;
- case KFREEBSD : host-os = freebsd ;
- case LINUX : host-os = linux ;
- case SUNOS :
- ECHO "SunOS is not a supported operating system." ;
- ECHO "We believe last version of SunOS was released in 1992, " ;
- ECHO "so if you get this message, something is very wrong with configuration logic. " ;
- ECHO "Please report this as a bug. " ;
- EXIT ;
- case * : host-os = unix ;
- }
- }
- return $(host-os:L) ;
-}
-
-
-# The two OS features define a known set of abstract OS names. The host-os is
-# the OS under which bjam is running. Even though this should really be a fixed
-# property we need to list all the values to prevent unknown value errors. Both
-# set the default value to the current OS to account for the default use case of
-# building on the target OS.
-feature.feature host-os : $(.os-names) ;
-feature.set-default host-os : [ default-host-os ] ;
-
-feature.feature target-os : $(.os-names) : propagated link-incompatible ;
-feature.set-default target-os : [ default-host-os ] ;
-
-
-feature.feature toolset : : implicit propagated symmetric ;
-feature.feature stdlib : native : propagated composite ;
-feature.feature link : shared static : propagated ;
-feature.feature runtime-link : shared static : propagated ;
-feature.feature runtime-debugging : on off : propagated ;
-feature.feature optimization : off speed space none : propagated ;
-feature.feature profiling : off on : propagated ;
-feature.feature inlining : off on full : propagated ;
-feature.feature threading : single multi : propagated ;
-feature.feature rtti : on off : propagated ;
-feature.feature exception-handling : on off : propagated ;
-
-# Whether there is support for asynchronous EH (e.g. catching SEGVs).
-feature.feature asynch-exceptions : off on : propagated ;
-
-# Whether all extern "C" functions are considered nothrow by default.
-feature.feature extern-c-nothrow : off on : propagated ;
-
-feature.feature debug-symbols : on off none : propagated ;
-# Controls whether the binary should be stripped -- that is have
-# everything not necessary to running removed. This option should
-# not be very often needed. Also, this feature will show up in
-# target paths of everything, not just binaries. Should fix that
-# when impelementing feature relevance.
-feature.feature strip : off on : propagated ;
-feature.feature define : : free ;
-feature.feature undef : : free ;
-feature.feature "include" : : free path ; #order-sensitive ;
-feature.feature cflags : : free ;
-feature.feature cxxflags : : free ;
-feature.feature fflags : : free ;
-feature.feature asmflags : : free ;
-feature.feature linkflags : : free ;
-feature.feature archiveflags : : free ;
-feature.feature version : : free ;
-
-# Generic, i.e. non-language specific, flags for tools.
-feature.feature flags : : free ;
-feature.feature location-prefix : : free ;
-
-
-# The following features are incidental since they have no effect on built
-# products. Not making them incidental will result in problems in corner cases,
-# e.g.:
-#
-# unit-test a : a.cpp : <use>b ;
-# lib b : a.cpp b ;
-#
-# Here, if <use> is not incidental, we would decide we have two targets for
-# a.obj with different properties and complain about it.
-#
-# Note that making a feature incidental does not mean it is ignored. It may be
-# ignored when creating a virtual target, but the rest of build process will use
-# them.
-feature.feature use : : free dependency incidental ;
-feature.feature dependency : : free dependency incidental ;
-feature.feature implicit-dependency : : free dependency incidental ;
-
-feature.feature warnings :
- on # Enable default/"reasonable" warning level for the tool.
- all # Enable all possible warnings issued by the tool.
- off # Disable all warnings issued by the tool.
- : incidental propagated ;
-
-feature.feature warnings-as-errors :
- off # Do not fail the compilation if there are warnings.
- on # Fail the compilation if there are warnings.
- : incidental propagated ;
-
-# Feature that allows us to configure the maximal template instantiation depth
-# level allowed by a C++ compiler. Applies only to C++ toolsets whose compilers
-# actually support this configuration setting.
-#
-# Note that Boost Build currently does not allow defining features that take any
-# positive integral value as a parameter, which is what we need here, so we just
-# define some of the values here and leave it up to the user to extend this set
-# as he needs using the feature.extend rule.
-#
-# TODO: This should be upgraded as soon as Boost Build adds support for custom
-# validated feature values or at least features allowing any positive integral
-# value. See related Boost Build related trac ticket #194.
-#
-feature.feature c++-template-depth
- :
- [ numbers.range 64 1024 : 64 ]
- [ numbers.range 20 1000 : 10 ]
- # Maximum template instantiation depth guaranteed for ANSI/ISO C++
- # conforming programs.
- 17
- :
- incidental optional propagated ;
-
-feature.feature source : : free dependency incidental ;
-feature.feature library : : free dependency incidental ;
-feature.feature file : : free dependency incidental ;
-feature.feature find-shared-library : : free ; #order-sensitive ;
-feature.feature find-static-library : : free ; #order-sensitive ;
-feature.feature library-path : : free path ; #order-sensitive ;
-
-# Internal feature.
-feature.feature library-file : : free dependency ;
-
-feature.feature name : : free ;
-feature.feature tag : : free ;
-feature.feature search : : free path ; #order-sensitive ;
-feature.feature location : : free path ;
-feature.feature dll-path : : free path ;
-feature.feature hardcode-dll-paths : true false : incidental ;
-
-
-# An internal feature that holds the paths of all dependency shared libraries.
-# On Windows, it is needed so that we can add all those paths to PATH when
-# running applications. On Linux, it is needed to add proper -rpath-link command
-# line options.
-feature.feature xdll-path : : free path ;
-
-# Provides means to specify def-file for windows DLLs.
-feature.feature def-file : : free dependency ;
-
-feature.feature suppress-import-lib : false true : incidental ;
-
-# Internal feature used to store the name of a bjam action to call when building
-# a target.
-feature.feature action : : free ;
-
-# This feature is used to allow specific generators to run. For example, QT
-# tools can only be invoked when QT library is used. In that case, <allow>qt
-# will be in usage requirement of the library.
-feature.feature allow : : free ;
-
-# The addressing model to generate code for. Currently a limited set only
-# specifying the bit size of pointers.
-feature.feature address-model : 16 32 64 32_64 : propagated optional ;
-
-# Type of CPU architecture to compile for.
-feature.feature architecture :
- # x86 and x86-64
- x86
-
- # ia64
- ia64
-
- # Sparc
- sparc
-
- # RS/6000 & PowerPC
- power
-
- # MIPS/SGI
- mips1 mips2 mips3 mips4 mips32 mips32r2 mips64
-
- # HP/PA-RISC
- parisc
-
- # Advanced RISC Machines
- arm
-
- # Combined architectures for platforms/toolsets that support building for
- # multiple architectures at once. "combined" would be the default multi-arch
- # for the toolset.
- combined
- combined-x86-power
-
- : propagated optional ;
-
-# The specific instruction set in an architecture to compile.
-feature.feature instruction-set :
- # x86 and x86-64
- native i486 i586 i686 pentium pentium-mmx pentiumpro pentium2 pentium3
- pentium3m pentium-m pentium4 pentium4m prescott nocona core2 corei7 corei7-avx core-avx-i
- conroe conroe-xe conroe-l allendale merom merom-xe kentsfield kentsfield-xe penryn wolfdale
- yorksfield nehalem sandy-bridge ivy-bridge haswell k6 k6-2 k6-3 athlon athlon-tbird athlon-4 athlon-xp
- athlon-mp k8 opteron athlon64 athlon-fx k8-sse3 opteron-sse3 athlon64-sse3 amdfam10 barcelona
- bdver1 bdver2 bdver3 btver1 btver2 winchip-c6 winchip2 c3 c3-2 atom
-
- # ia64
- itanium itanium1 merced itanium2 mckinley
-
- # Sparc
- v7 cypress v8 supersparc sparclite hypersparc sparclite86x f930 f934
- sparclet tsc701 v9 ultrasparc ultrasparc3
-
- # RS/6000 & PowerPC
- 401 403 405 405fp 440 440fp 505 601 602 603 603e 604 604e 620 630 740 7400
- 7450 750 801 821 823 860 970 8540 power-common ec603e g3 g4 g5 power power2
- power3 power4 power5 powerpc powerpc64 rios rios1 rsc rios2 rs64a
-
- # MIPS
- 4kc 4kp 5kc 20kc m4k r2000 r3000 r3900 r4000 r4100 r4300 r4400 r4600 r4650
- r6000 r8000 rm7000 rm9000 orion sb1 vr4100 vr4111 vr4120 vr4130 vr4300
- vr5000 vr5400 vr5500
-
- # HP/PA-RISC
- 700 7100 7100lc 7200 7300 8000
-
- # Advanced RISC Machines
- armv2 armv2a armv3 armv3m armv4 armv4t armv5 armv5t armv5te armv6 armv6j iwmmxt ep9312
- armv7 armv7s
-
- : propagated optional ;
-
-# Used to select a specific variant of C++ ABI if the compiler supports several.
-feature.feature c++abi : : propagated optional ;
-
-feature.feature conditional : : incidental free ;
-
-# The value of 'no' prevents building of a target.
-feature.feature build : yes no : optional ;
-
-# Windows-specific features
-
-feature.feature user-interface : console gui wince native auto ;
-
-feature.feature variant : : implicit composite propagated symmetric ;
-
-
-# Declares a new variant.
-#
-# First determines explicit properties for this variant, by refining parents'
-# explicit properties with the passed explicit properties. The result is
-# remembered and will be used if this variant is used as parent.
-#
-# Second, determines the full property set for this variant by adding to the
-# explicit properties default values for all missing non-symmetric properties.
-#
-# Lastly, makes appropriate value of 'variant' property expand to the full
-# property set.
-#
-rule variant ( name # Name of the variant
- : parents-or-properties * # Specifies parent variants, if
- # 'explicit-properties' are given, and
- # explicit-properties or parents otherwise.
- : explicit-properties * # Explicit properties.
- )
-{
- local parents ;
- if ! $(explicit-properties)
- {
- if $(parents-or-properties[1]:G)
- {
- explicit-properties = $(parents-or-properties) ;
- }
- else
- {
- parents = $(parents-or-properties) ;
- }
- }
- else
- {
- parents = $(parents-or-properties) ;
- }
-
- # The problem is that we have to check for conflicts between base variants.
- if $(parents[2])
- {
- errors.error "multiple base variants are not yet supported" ;
- }
-
- local inherited ;
- # Add explicitly specified properties for parents.
- for local p in $(parents)
- {
- # TODO: This check may be made stricter.
- if ! [ feature.is-implicit-value $(p) ]
- {
- errors.error "Invalid base variant" $(p) ;
- }
-
- inherited += $(.explicit-properties.$(p)) ;
- }
- property.validate $(explicit-properties) ;
- explicit-properties = [ property.refine $(inherited)
- : $(explicit-properties) ] ;
-
- # Record explicitly specified properties for this variant. We do this after
- # inheriting parents' properties so they affect other variants derived from
- # this one.
- .explicit-properties.$(name) = $(explicit-properties) ;
-
- feature.extend variant : $(name) ;
- feature.compose <variant>$(name) : $(explicit-properties) ;
-}
-IMPORT $(__name__) : variant : : variant ;
-
-
-variant debug : <optimization>off <debug-symbols>on <inlining>off
- <runtime-debugging>on ;
-variant release : <optimization>speed <debug-symbols>off <inlining>full
- <runtime-debugging>off <define>NDEBUG ;
-variant profile : release : <profiling>on <debug-symbols>on ;
-
-
-class searched-lib-target : abstract-file-target
-{
- rule __init__ ( name
- : project
- : shared ?
- : search *
- : action
- )
- {
- abstract-file-target.__init__ $(name) : SEARCHED_LIB : $(project)
- : $(action) : ;
-
- self.shared = $(shared) ;
- self.search = $(search) ;
- }
-
- rule shared ( )
- {
- return $(self.shared) ;
- }
-
- rule search ( )
- {
- return $(self.search) ;
- }
-
- rule actualize-location ( target )
- {
- NOTFILE $(target) ;
- }
-
- rule path ( )
- {
- }
-}
-
-
-# The generator class for libraries (target type LIB). Depending on properties
-# it will request building of the appropriate specific library type --
-# -- SHARED_LIB, STATIC_LIB or SHARED_LIB.
-#
-class lib-generator : generator
-{
- rule __init__ ( * : * )
- {
- generator.__init__ $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8)
- : $(9) : $(10) : $(11) : $(12) : $(13) : $(14) : $(15) : $(16) :
- $(17) : $(18) : $(19) ;
- }
-
- rule run ( project name ? : property-set : sources * )
- {
- # The lib generator is composing, and can be only invoked with an
- # explicit name. This check is present in generator.run (and so in
- # builtin.linking-generator) but duplicated here to avoid doing extra
- # work.
- if $(name)
- {
- local properties = [ $(property-set).raw ] ;
- # Determine the needed target type.
- local actual-type ;
- # <source>files can be generated by <conditional>@rule feature
- # in which case we do not consider it a SEARCHED_LIB type.
- if ! <source> in $(properties:G) &&
- ( <search> in $(properties:G) || <name> in $(properties:G) )
- {
- actual-type = SEARCHED_LIB ;
- }
- else if <file> in $(properties:G)
- {
- actual-type = LIB ;
- }
- else if <link>shared in $(properties)
- {
- actual-type = SHARED_LIB ;
- }
- else
- {
- actual-type = STATIC_LIB ;
- }
- property-set = [ $(property-set).add-raw <main-target-type>LIB ] ;
- # Construct the target.
- return [ generators.construct $(project) $(name) : $(actual-type)
- : $(property-set) : $(sources) ] ;
- }
- }
-
- rule viable-source-types ( )
- {
- return * ;
- }
-}
-
-
-generators.register [ new lib-generator builtin.lib-generator : : LIB ] ;
-
-
-# The implementation of the 'lib' rule. Beyond standard syntax that rule allows
-# simplified: "lib a b c ;".
-#
-rule lib ( names + : sources * : requirements * : default-build * :
- usage-requirements * )
-{
- if $(names[2])
- {
- if <name> in $(requirements:G)
- {
- errors.user-error "When several names are given to the 'lib' rule" :
- "it is not allowed to specify the <name> feature." ;
- }
- if $(sources)
- {
- errors.user-error "When several names are given to the 'lib' rule" :
- "it is not allowed to specify sources." ;
- }
- }
-
- # This is a circular module dependency so it must be imported here.
- import targets ;
-
- local project = [ project.current ] ;
- local result ;
-
- for local name in $(names)
- {
- local r = $(requirements) ;
- # Support " lib a ; " and " lib a b c ; " syntax.
- if ! $(sources) && ! <name> in $(requirements:G)
- && ! <file> in $(requirements:G)
- {
- r += <name>$(name) ;
- }
- result += [ targets.main-target-alternative
- [ new typed-target $(name) : $(project) : LIB
- : [ targets.main-target-sources $(sources) : $(name) ]
- : [ targets.main-target-requirements $(r) : $(project) ]
- : [ targets.main-target-default-build $(default-build) : $(project) ]
- : [ targets.main-target-usage-requirements $(usage-requirements) : $(project) ]
- ] ] ;
- }
- return $(result) ;
-}
-IMPORT $(__name__) : lib : : lib ;
-
-
-class searched-lib-generator : generator
-{
- import property-set ;
-
- rule __init__ ( )
- {
- # The requirements cause the generators to be tried *only* when we are
- # building a lib target with a 'search' feature. This seems ugly --- all
- # we want is to make sure searched-lib-generator is not invoked deep
- # inside transformation search to produce intermediate targets.
- generator.__init__ searched-lib-generator : : SEARCHED_LIB ;
- }
-
- rule run ( project name ? : property-set : sources * )
- {
- if $(name)
- {
- # If 'name' is empty, it means we have not been called to build a
- # top-level target. In this case, we just fail immediately, because
- # searched-lib-generator cannot be used to produce intermediate
- # targets.
-
- local properties = [ $(property-set).raw ] ;
- local shared ;
- if <link>shared in $(properties)
- {
- shared = true ;
- }
-
- local search = [ feature.get-values <search> : $(properties) ] ;
-
- local a = [ new null-action $(property-set) ] ;
- local lib-name = [ feature.get-values <name> : $(properties) ] ;
- lib-name ?= $(name) ;
- local t = [ new searched-lib-target $(lib-name) : $(project)
- : $(shared) : $(search) : $(a) ] ;
- # We return sources for a simple reason. If there is
- # lib png : z : <name>png ;
- # the 'z' target should be returned, so that apps linking to 'png'
- # will link to 'z', too.
- return [ property-set.create <xdll-path>$(search) ]
- [ virtual-target.register $(t) ] $(sources) ;
- }
- }
-}
-
-generators.register [ new searched-lib-generator ] ;
-
-
-class prebuilt-lib-generator : generator
-{
- rule __init__ ( * : * )
- {
- generator.__init__ $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8)
- : $(9) : $(10) : $(11) : $(12) : $(13) : $(14) : $(15) : $(16) :
- $(17) : $(18) : $(19) ;
- }
-
- rule run ( project name ? : property-set : sources * )
- {
- local f = [ $(property-set).get <file> ] ;
- return $(f) $(sources) ;
- }
-}
-
-generators.register
- [ new prebuilt-lib-generator builtin.prebuilt : : LIB : <file> ] ;
-
-generators.override builtin.prebuilt : builtin.lib-generator ;
-
-class preprocessed-target-class : basic-target
-{
- import generators ;
- rule construct ( name : sources * : property-set )
- {
- local result = [ generators.construct [ project ]
- $(name) : PREPROCESSED_CPP : $(property-set) : $(sources) ] ;
- if ! $(result)
- {
- result = [ generators.construct [ project ]
- $(name) : PREPROCESSED_C : $(property-set) : $(sources) ] ;
- }
- if ! $(result)
- {
- local s ;
- for x in $(sources)
- {
- s += [ $(x).name ] ;
- }
- local p = [ project ] ;
- errors.user-error
- "In project" [ $(p).name ] :
- "Could not construct preprocessed file \"$(name)\" from $(s:J=, )." ;
- }
- return $(result) ;
- }
-}
-
-rule preprocessed ( name : sources * : requirements * : default-build * :
- usage-requirements * )
-{
- local project = [ project.current ] ;
- return [ targets.main-target-alternative
- [ new preprocessed-target-class $(name) : $(project)
- : [ targets.main-target-sources $(sources) : $(name) ]
- : [ targets.main-target-requirements $(requirements) : $(project) ]
- : [ targets.main-target-default-build $(default-build) : $(project) ]
- : [ targets.main-target-usage-requirements $(usage-requirements) : $(project) ]
- ] ] ;
-}
-
-IMPORT $(__name__) : preprocessed : : preprocessed ;
-
-class compile-action : action
-{
- import sequence ;
-
- rule __init__ ( targets * : sources * : action-name : properties * )
- {
- action.__init__ $(targets) : $(sources) : $(action-name) : $(properties) ;
- }
-
- # For all virtual targets for the same dependency graph as self, i.e. which
- # belong to the same main target, add their directories to the include path.
- #
- rule adjust-properties ( property-set )
- {
- local s = [ $(self.targets[1]).creating-subvariant ] ;
- if $(s)
- {
- return [ $(property-set).add-raw
- [ $(s).implicit-includes "include" : H ] ] ;
- }
- else
- {
- return $(property-set) ;
- }
- }
-}
-
-
-# Declare a special compiler generator. The only thing it does is changing the
-# type used to represent 'action' in the constructed dependency graph to
-# 'compile-action'. That class in turn adds additional include paths to handle
-# cases when a source file includes headers which are generated themselves.
-#
-class C-compiling-generator : generator
-{
- rule __init__ ( id : source-types + : target-types + : requirements *
- : optional-properties * )
- {
- generator.__init__ $(id) : $(source-types) : $(target-types) :
- $(requirements) : $(optional-properties) ;
- }
-
- rule action-class ( )
- {
- return compile-action ;
- }
-}
-
-
-rule register-c-compiler ( id : source-types + : target-types + : requirements *
- : optional-properties * )
-{
- generators.register [ new C-compiling-generator $(id) : $(source-types) :
- $(target-types) : $(requirements) : $(optional-properties) ] ;
-}
-
-# FIXME: this is ugly, should find a better way (we would like client code to
-# register all generators as "generators.some-rule" instead of
-# "some-module.some-rule".)
-#
-IMPORT $(__name__) : register-c-compiler : : generators.register-c-compiler ;
-
-
-# The generator class for handling EXE and SHARED_LIB creation.
-#
-class linking-generator : generator
-{
- import path ;
- import project ;
- import property-set ;
- import type ;
-
- rule __init__ ( id
- composing ? : # The generator will be composing if a non-empty
- # string is passed or the parameter is not given. To
- # make the generator non-composing, pass an empty
- # string ("").
- source-types + :
- target-types + :
- requirements * )
- {
- composing ?= true ;
- generator.__init__ $(id) $(composing) : $(source-types)
- : $(target-types) : $(requirements) ;
- }
-
- rule run ( project name ? : property-set : sources + )
- {
- sources += [ $(property-set).get <library> ] ;
-
- # Add <library-path> properties for all searched libraries.
- local extra ;
- for local s in $(sources)
- {
- if [ $(s).type ] = SEARCHED_LIB
- {
- local search = [ $(s).search ] ;
- extra += <library-path>$(search) ;
- }
- }
-
- # It is possible that sources include shared libraries that did not came
- # from 'lib' targets, e.g. .so files specified as sources. In this case
- # we have to add extra dll-path properties and propagate extra xdll-path
- # properties so that application linking to us will get xdll-path to
- # those libraries.
- local extra-xdll-paths ;
- for local s in $(sources)
- {
- if [ type.is-derived [ $(s).type ] SHARED_LIB ] && ! [ $(s).action ]
- {
- # Unfortunately, we do not have a good way to find the path to a
- # file, so use this nasty approach.
- #
- # TODO: This needs to be done better. One thing that is really
- # broken with this is that it does not work correctly with
- # projects having multiple source locations.
- local p = [ $(s).project ] ;
- local location = [ path.root [ $(s).name ]
- [ $(p).get source-location ] ] ;
- extra-xdll-paths += [ path.parent $(location) ] ;
- }
- }
-
- # Hardcode DLL paths only when linking executables.
- # Pros: do not need to relink libraries when installing.
- # Cons: "standalone" libraries (plugins, python extensions) can not
- # hardcode paths to dependent libraries.
- if [ $(property-set).get <hardcode-dll-paths> ] = true
- && [ type.is-derived $(self.target-types[1]) EXE ]
- {
- local xdll-path = [ $(property-set).get <xdll-path> ] ;
- extra += <dll-path>$(xdll-path) <dll-path>$(extra-xdll-paths) ;
- }
-
- if $(extra)
- {
- property-set = [ $(property-set).add-raw $(extra) ] ;
- }
-
- local result = [ generator.run $(project) $(name) : $(property-set)
- : $(sources) ] ;
-
- local ur ;
- if $(result)
- {
- ur = [ extra-usage-requirements $(result) : $(property-set) ] ;
- ur = [ $(ur).add
- [ property-set.create <xdll-path>$(extra-xdll-paths) ] ] ;
- }
- return $(ur) $(result) ;
- }
-
- rule extra-usage-requirements ( created-targets * : property-set )
- {
- local result = [ property-set.empty ] ;
- local extra ;
-
- # Add appropricate <xdll-path> usage requirements.
- local raw = [ $(property-set).raw ] ;
- if <link>shared in $(raw)
- {
- local paths ;
- local pwd = [ path.pwd ] ;
- for local t in $(created-targets)
- {
- if [ type.is-derived [ $(t).type ] SHARED_LIB ]
- {
- paths += [ path.root [ path.make [ $(t).path ] ] $(pwd) ] ;
- }
- }
- extra += $(paths:G=<xdll-path>) ;
- }
-
- # We need to pass <xdll-path> features that we've got from sources,
- # because if a shared library is built, exe using it needs to know paths
- # to other shared libraries this one depends on in order to be able to
- # find them all at runtime.
-
- # Just pass all features in property-set, it is theorically possible
- # that we will propagate <xdll-path> features explicitly specified by
- # the user, but then the user is to blaim for using an internal feature.
- local values = [ $(property-set).get <xdll-path> ] ;
- extra += $(values:G=<xdll-path>) ;
-
- if $(extra)
- {
- result = [ property-set.create $(extra) ] ;
- }
- return $(result) ;
- }
-
- rule generated-targets ( sources + : property-set : project name ? )
- {
- local sources2 ; # Sources to pass to inherited rule.
- local properties2 ; # Properties to pass to inherited rule.
- local libraries ; # Library sources.
-
- # Searched libraries are not passed as arguments to the linker but via
- # some option. So, we pass them to the action using a property.
- properties2 = [ $(property-set).raw ] ;
- local fsa ;
- local fst ;
- for local s in $(sources)
- {
- if [ type.is-derived [ $(s).type ] SEARCHED_LIB ]
- {
- local name = [ $(s).name ] ;
- if [ $(s).shared ]
- {
- fsa += $(name) ;
- }
- else
- {
- fst += $(name) ;
- }
- }
- else
- {
- sources2 += $(s) ;
- }
- }
- properties2 += <find-shared-library>$(fsa:J=&&)
- <find-static-library>$(fst:J=&&) ;
-
- return [ generator.generated-targets $(sources2)
- : [ property-set.create $(properties2) ] : $(project) $(name) ] ;
- }
-}
-
-
-rule register-linker ( id composing ? : source-types + : target-types +
- : requirements * )
-{
- generators.register [ new linking-generator $(id) $(composing)
- : $(source-types) : $(target-types) : $(requirements) ] ;
-}
-
-
-# The generator class for handling STATIC_LIB creation.
-#
-class archive-generator : generator
-{
- import property-set ;
-
- rule __init__ ( id composing ? : source-types + : target-types +
- : requirements * )
- {
- composing ?= true ;
- generator.__init__ $(id) $(composing) : $(source-types)
- : $(target-types) : $(requirements) ;
- }
-
- rule run ( project name ? : property-set : sources + )
- {
- sources += [ $(property-set).get <library> ] ;
-
- local result = [ generator.run $(project) $(name) : $(property-set)
- : $(sources) ] ;
-
- # For static linking, if we get a library in source, we can not directly
- # link to it so we need to cause our dependencies to link to that
- # library. There are two approaches:
- # - adding the library to the list of returned targets.
- # - using the <library> usage requirements.
- # The problem with the first is:
- #
- # lib a1 : : <file>liba1.a ;
- # lib a2 : a2.cpp a1 : <link>static ;
- # install dist : a2 ;
- #
- # here we will try to install 'a1', even though it is not necessary in
- # the general case. With the second approach, even indirect dependants
- # will link to the library, but it should not cause any harm. So, return
- # all LIB sources together with created targets, so that dependants link
- # to them.
- local usage-requirements ;
- if [ $(property-set).get <link> ] = static
- {
- for local t in $(sources)
- {
- if [ type.is-derived [ $(t).type ] LIB ]
- {
- usage-requirements += <library>$(t) ;
- }
- }
- }
-
- usage-requirements = [ property-set.create $(usage-requirements) ] ;
-
- return $(usage-requirements) $(result) ;
- }
-}
-
-
-rule register-archiver ( id composing ? : source-types + : target-types +
- : requirements * )
-{
- generators.register [ new archive-generator $(id) $(composing)
- : $(source-types) : $(target-types) : $(requirements) ] ;
-}
-
-
-# Generator that accepts everything and produces nothing. Useful as a general
-# fallback for toolset-specific actions like PCH generation.
-#
-class dummy-generator : generator
-{
- import property-set ;
-
- rule run ( project name ? : property-set : sources + )
- {
- return [ property-set.empty ] ;
- }
-}
-
-IMPORT $(__name__) : register-linker register-archiver
- : : generators.register-linker generators.register-archiver ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/cast.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/cast.jam b/ext/kenlm/jam-files/boost-build/tools/cast.jam
deleted file mode 100644
index 41b0ac3..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/cast.jam
+++ /dev/null
@@ -1,91 +0,0 @@
-# Copyright 2005 Vladimir Prus.
-# Distributed under the Boost Software License, Version 1.0. (See
-# accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# Defines main target 'cast', used to change type for target. For example, in Qt
-# library one wants two kinds of CPP files -- those that just compiled and those
-# that are passed via the MOC tool.
-#
-# This is done with:
-#
-# exe main : main.cpp [ cast _ moccable-cpp : widget.cpp ] ;
-#
-# Boost.Build will assign target type CPP to both main.cpp and widget.cpp. Then,
-# the cast rule will change target type of widget.cpp to MOCCABLE-CPP, and Qt
-# support will run the MOC tool as part of the build process.
-#
-# At the moment, the 'cast' rule only works for non-derived (source) targets.
-#
-# TODO: The following comment is unclear or incorrect. Clean it up.
-# > Another solution would be to add a separate main target 'moc-them' that
-# > would moc all the passed sources, no matter what their type is, but I prefer
-# > cast, as defining a new target type + generator for that type is somewhat
-# > simpler than defining a main target rule.
-
-import "class" : new ;
-import project ;
-import property-set ;
-import targets ;
-import type ;
-
-
-class cast-target-class : typed-target
-{
- import type ;
-
- rule __init__ ( name : project : type : sources * : requirements * :
- default-build * : usage-requirements * )
- {
- typed-target.__init__ $(name) : $(project) : $(type) : $(sources) :
- $(requirements) : $(default-build) : $(usage-requirements) ;
- }
-
- rule construct ( name : source-targets * : property-set )
- {
- local result ;
- for local s in $(source-targets)
- {
- if ! [ class.is-a $(s) : file-target ]
- {
- import errors : user-error : errors.user-error ;
- errors.user-error Source to the 'cast' rule is not a file! ;
- }
- if [ $(s).action ]
- {
- import errors : user-error : errors.user-error ;
- errors.user-error Only non-derived target are allowed for
- 'cast'. : when building [ full-name ] ;
- }
- local r = [ $(s).clone-with-different-type $(self.type) ] ;
- result += [ virtual-target.register $(r) ] ;
- }
- return [ property-set.empty ] $(result) ;
- }
-}
-
-
-rule cast ( name type : sources * : requirements * : default-build * :
- usage-requirements * )
-{
- local project = [ project.current ] ;
-
- local real-type = [ type.type-from-rule-name $(type) ] ;
- if ! $(real-type)
- {
- import errors ;
- errors.user-error No type corresponds to the main target rule name
- '$(type)' : "Hint: try a lowercase name" ;
- }
-
- targets.main-target-alternative [ new cast-target-class $(name) : $(project)
- : $(real-type)
- : [ targets.main-target-sources $(sources) : $(name) ]
- : [ targets.main-target-requirements $(requirements) : $(project) ]
- : [ targets.main-target-default-build $(default-build) : $(project) ]
- : [ targets.main-target-usage-requirements $(usage-requirements) :
- $(project) ] ] ;
-}
-
-
-IMPORT $(__name__) : cast : : cast ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/clang-darwin.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/clang-darwin.jam b/ext/kenlm/jam-files/boost-build/tools/clang-darwin.jam
deleted file mode 100644
index a8abc7d..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/clang-darwin.jam
+++ /dev/null
@@ -1,170 +0,0 @@
-# Copyright Vladimir Prus 2004.
-# Copyright Noel Belcourt 2007.
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt
-# or copy at http://www.boost.org/LICENSE_1_0.txt)
-
-import clang ;
-import feature : feature ;
-import os ;
-import toolset ;
-import toolset : flags ;
-import gcc ;
-import common ;
-import errors ;
-import generators ;
-
-feature.extend-subfeature toolset clang : platform : darwin ;
-
-toolset.inherit-generators clang-darwin
- <toolset>clang <toolset-clang:platform>darwin
- : gcc
- # Don't inherit PCH generators. They were not tested, and probably
- # don't work for this compiler.
- : gcc.mingw.link gcc.mingw.link.dll gcc.compile.c.pch gcc.compile.c++.pch
- ;
-
-generators.override clang-darwin.prebuilt : builtin.lib-generator ;
-generators.override clang-darwin.prebuilt : builtin.prebuilt ;
-generators.override clang-darwin.searched-lib-generator : searched-lib-generator ;
-
-toolset.inherit-rules clang-darwin : gcc ;
-toolset.inherit-flags clang-darwin : gcc
- : <inlining>off <inlining>on <inlining>full <optimization>space
- <warnings>off <warnings>all <warnings>on
- <architecture>x86/<address-model>32
- <architecture>x86/<address-model>64
- ;
-
-if [ MATCH (--debug-configuration) : [ modules.peek : ARGV ] ]
-{
- .debug-configuration = true ;
-}
-
-# vectorization diagnostics
-feature vectorize : off on full ;
-
-# Initializes the clang-darwin toolset
-# version in optional
-# name (default clang++) is used to invoke the specified clang complier
-# compile and link options allow you to specify addition command line options for each version
-rule init ( version ? : command * : options * )
-{
- command = [ common.get-invocation-command clang-darwin : clang++
- : $(command) ] ;
-
- # Determine the version
- local command-string = $(command:J=" ") ;
- if $(command)
- {
- version ?= [ MATCH "^([0-9.]+)"
- : [ SHELL "$(command-string) -dumpversion" ] ] ;
- }
-
- local condition = [ common.check-init-parameters clang-darwin
- : version $(version) ] ;
-
- common.handle-options clang-darwin : $(condition) : $(command) : $(options) ;
-
- gcc.init-link-flags clang-darwin darwin $(condition) ;
-
-}
-
-SPACE = " " ;
-
-flags clang-darwin.compile OPTIONS <cflags> ;
-flags clang-darwin.compile OPTIONS <cxxflags> ;
-# flags clang-darwin.compile INCLUDES <include> ;
-
-# Declare flags and action for compilation.
-toolset.flags clang-darwin.compile OPTIONS <optimization>off : -O0 ;
-toolset.flags clang-darwin.compile OPTIONS <optimization>speed : -O3 ;
-toolset.flags clang-darwin.compile OPTIONS <optimization>space : -Os ;
-
-toolset.flags clang-darwin.compile OPTIONS <inlining>off : -fno-inline ;
-toolset.flags clang-darwin.compile OPTIONS <inlining>on : -Wno-inline ;
-toolset.flags clang-darwin.compile OPTIONS <inlining>full : -finline-functions -Wno-inline ;
-
-toolset.flags clang-darwin.compile OPTIONS <warnings>off : -w ;
-toolset.flags clang-darwin.compile OPTIONS <warnings>on : -Wall ;
-toolset.flags clang-darwin.compile OPTIONS <warnings>all : -Wall -pedantic ;
-toolset.flags clang-darwin.compile OPTIONS <warnings-as-errors>on : -Werror ;
-
-toolset.flags clang-darwin.compile OPTIONS <debug-symbols>on : -g ;
-toolset.flags clang-darwin.compile OPTIONS <profiling>on : -pg ;
-toolset.flags clang-darwin.compile OPTIONS <rtti>off : -fno-rtti ;
-
-actions compile.c
-{
- "$(CONFIG_COMMAND)" -x c $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
-}
-
-actions compile.c++
-{
- "$(CONFIG_COMMAND)" -x c++ $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
-}
-
-flags clang-darwin ARFLAGS <archiveflags> ;
-
-# Default value. Mostly for the sake of clang-linux
-# that inherits from gcc, but does not has the same
-# logic to set the .AR variable. We can put the same
-# logic in clang-linux, but that's hardly worth the trouble
-# as on Linux, 'ar' is always available.
-.AR = ar ;
-
-rule archive ( targets * : sources * : properties * )
-{
- # Always remove archive and start again. Here's rationale from
- # Andre Hentz:
- #
- # I had a file, say a1.c, that was included into liba.a.
- # I moved a1.c to a2.c, updated my Jamfiles and rebuilt.
- # My program was crashing with absurd errors.
- # After some debugging I traced it back to the fact that a1.o was *still*
- # in liba.a
- #
- # Rene Rivera:
- #
- # Originally removing the archive was done by splicing an RM
- # onto the archive action. That makes archives fail to build on NT
- # when they have many files because it will no longer execute the
- # action directly and blow the line length limit. Instead we
- # remove the file in a different action, just before the building
- # of the archive.
- #
- local clean.a = $(targets[1])(clean) ;
- TEMPORARY $(clean.a) ;
- NOCARE $(clean.a) ;
- LOCATE on $(clean.a) = [ on $(targets[1]) return $(LOCATE) ] ;
- DEPENDS $(clean.a) : $(sources) ;
- DEPENDS $(targets) : $(clean.a) ;
- common.RmTemps $(clean.a) : $(targets) ;
-}
-
-actions piecemeal archive
-{
- "$(.AR)" $(AROPTIONS) rc "$(<)" "$(>)"
- "ranlib" -cs "$(<)"
-}
-
-flags clang-darwin.link USER_OPTIONS <linkflags> ;
-
-# Declare actions for linking
-rule link ( targets * : sources * : properties * )
-{
- SPACE on $(targets) = " " ;
- # Serialize execution of the 'link' action, since
- # running N links in parallel is just slower.
- JAM_SEMAPHORE on $(targets) = <s>clang-darwin-link-semaphore ;
-}
-
-actions link bind LIBRARIES
-{
- "$(CONFIG_COMMAND)" $(USER_OPTIONS) -L"$(LINKPATH)" -o "$(<)" "$(>)" "$(LIBRARIES)" -l$(FINDLIBS-SA) -l$(FINDLIBS-ST) $(OPTIONS)
-}
-
-actions link.dll bind LIBRARIES
-{
- "$(CONFIG_COMMAND)" $(USER_OPTIONS) -L"$(LINKPATH)" -o "$(<)" -single_module -dynamiclib -install_name "$(<[1]:D=)" "$(>)" "$(LIBRARIES)" -l$(FINDLIBS-SA) -l$(FINDLIBS-ST) $(OPTIONS)
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/clang-linux.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/clang-linux.jam b/ext/kenlm/jam-files/boost-build/tools/clang-linux.jam
deleted file mode 100644
index 2999c2d..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/clang-linux.jam
+++ /dev/null
@@ -1,196 +0,0 @@
-# Copyright (c) 2003 Michael Stevens
-# Copyright (c) 2010-2011 Bryce Lelbach (blelbach@cct.lsu.edu, maintainer)
-#
-# Use, modification and distribution is subject to the Boost Software
-# License Version 1.0. (See accompanying file LICENSE_1_0.txt or
-# http://www.boost.org/LICENSE_1_0.txt)
-
-import toolset ;
-import feature ;
-import toolset : flags ;
-
-import clang ;
-import gcc ;
-import common ;
-import errors ;
-import generators ;
-import type ;
-import numbers ;
-
-feature.extend-subfeature toolset clang : platform : linux ;
-
-toolset.inherit-generators clang-linux
- <toolset>clang <toolset-clang:platform>linux : gcc
- : gcc.mingw.link gcc.mingw.link.dll gcc.cygwin.link gcc.cygwin.link.dll ;
-generators.override clang-linux.prebuilt : builtin.lib-generator ;
-generators.override clang-linux.prebuilt : builtin.prebuilt ;
-generators.override clang-linux.searched-lib-generator : searched-lib-generator ;
-
-# Override default do-nothing generators.
-generators.override clang-linux.compile.c.pch : pch.default-c-pch-generator ;
-generators.override clang-linux.compile.c++.pch : pch.default-cpp-pch-generator ;
-
-type.set-generated-target-suffix PCH
- : <toolset>clang <toolset-clang:platform>linux : pth ;
-
-toolset.inherit-rules clang-linux : gcc ;
-toolset.inherit-flags clang-linux : gcc
- : <inlining>off <inlining>on <inlining>full
- <optimization>space <optimization>speed
- <warnings>off <warnings>all <warnings>on ;
-
-if [ MATCH (--debug-configuration) : [ modules.peek : ARGV ] ] {
- .debug-configuration = true ;
-}
-
-rule init ( version ? : command * : options * ) {
- command = [ common.get-invocation-command clang-linux : clang++
- : $(command) ] ;
-
- # Determine the version
- local command-string = $(command:J=" ") ;
-
- if $(command) {
- version ?= [ MATCH "version ([0-9.]+)"
- : [ SHELL "$(command-string) --version" ] ] ;
- }
-
- local condition = [ common.check-init-parameters clang-linux
- : version $(version) ] ;
-
- common.handle-options clang-linux : $(condition) : $(command) : $(options) ;
-
- gcc.init-link-flags clang-linux gnu $(condition) ;
-}
-
-###############################################################################
-# Flags
-
-toolset.flags clang-linux.compile OPTIONS <cflags> ;
-toolset.flags clang-linux.compile.c++ OPTIONS <cxxflags> ;
-
-toolset.flags clang-linux.compile OPTIONS <optimization>off : ;
-toolset.flags clang-linux.compile OPTIONS <optimization>speed : -O3 ;
-toolset.flags clang-linux.compile OPTIONS <optimization>space : -Os ;
-
-# note: clang silently ignores some of these inlining options
-toolset.flags clang-linux.compile OPTIONS <inlining>off : -fno-inline ;
-toolset.flags clang-linux.compile OPTIONS <inlining>on : -Wno-inline ;
-toolset.flags clang-linux.compile OPTIONS <inlining>full : -finline-functions -Wno-inline ;
-
-toolset.flags clang-linux.compile OPTIONS <warnings>off : -w ;
-toolset.flags clang-linux.compile OPTIONS <warnings>on : -Wall ;
-toolset.flags clang-linux.compile OPTIONS <warnings>all : -Wall -pedantic ;
-toolset.flags clang-linux.compile OPTIONS <warnings-as-errors>on : -Werror ;
-
-toolset.flags clang-linux.compile OPTIONS <debug-symbols>on : -g ;
-toolset.flags clang-linux.compile OPTIONS <profiling>on : -pg ;
-toolset.flags clang-linux.compile OPTIONS <rtti>off : -fno-rtti ;
-
-###############################################################################
-# C and C++ compilation
-
-rule compile.c++ ( targets * : sources * : properties * ) {
- gcc.setup-threading $(targets) : $(sources) : $(properties) ;
- gcc.setup-fpic $(targets) : $(sources) : $(properties) ;
- gcc.setup-address-model $(targets) : $(sources) : $(properties) ;
-
- local pth-file = [ on $(<) return $(PCH_FILE) ] ;
-
- if $(pth-file) {
- DEPENDS $(<) : $(pth-file) ;
- clang-linux.compile.c++.with-pch $(targets) : $(sources) ;
- }
- else {
- clang-linux.compile.c++.without-pth $(targets) : $(sources) ;
- }
-}
-
-actions compile.c++.without-pth {
- "$(CONFIG_COMMAND)" -c -x c++ $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -o "$(<)" "$(>)"
-}
-
-actions compile.c++.with-pch bind PCH_FILE
-{
- "$(CONFIG_COMMAND)" -c -x c++ $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -Xclang -include-pth -Xclang "$(PCH_FILE)" -o "$(<)" "$(>)"
-}
-
-rule compile.c ( targets * : sources * : properties * )
-{
- gcc.setup-threading $(targets) : $(sources) : $(properties) ;
- gcc.setup-fpic $(targets) : $(sources) : $(properties) ;
- gcc.setup-address-model $(targets) : $(sources) : $(properties) ;
-
- local pth-file = [ on $(<) return $(PCH_FILE) ] ;
-
- if $(pth-file) {
- DEPENDS $(<) : $(pth-file) ;
- clang-linux.compile.c.with-pch $(targets) : $(sources) ;
- }
- else {
- clang-linux.compile.c.without-pth $(targets) : $(sources) ;
- }
-}
-
-actions compile.c.without-pth
-{
- "$(CONFIG_COMMAND)" -c -x c $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
-}
-
-actions compile.c.with-pch bind PCH_FILE
-{
- "$(CONFIG_COMMAND)" -c -x c $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -Xclang -include-pth -Xclang "$(PCH_FILE)" -c -o "$(<)" "$(>)"
-}
-
-###############################################################################
-# PCH emission
-
-rule compile.c++.pch ( targets * : sources * : properties * ) {
- gcc.setup-threading $(targets) : $(sources) : $(properties) ;
- gcc.setup-fpic $(targets) : $(sources) : $(properties) ;
- gcc.setup-address-model $(targets) : $(sources) : $(properties) ;
-}
-
-actions compile.c++.pch {
- rm -f "$(<)" && "$(CONFIG_COMMAND)" -x c++-header $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -Xclang -emit-pth -o "$(<)" "$(>)"
-}
-
-rule compile.c.pch ( targets * : sources * : properties * ) {
- gcc.setup-threading $(targets) : $(sources) : $(properties) ;
- gcc.setup-fpic $(targets) : $(sources) : $(properties) ;
- gcc.setup-address-model $(targets) : $(sources) : $(properties) ;
-}
-
-actions compile.c.pch
-{
- rm -f "$(<)" && "$(CONFIG_COMMAND)" -x c-header $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -Xclang -emit-pth -o "$(<)" "$(>)"
-}
-
-###############################################################################
-# Linking
-
-SPACE = " " ;
-
-rule link ( targets * : sources * : properties * ) {
- gcc.setup-threading $(targets) : $(sources) : $(properties) ;
- gcc.setup-address-model $(targets) : $(sources) : $(properties) ;
- SPACE on $(targets) = " " ;
- JAM_SEMAPHORE on $(targets) = <s>clang-linux-link-semaphore ;
-}
-
-actions link bind LIBRARIES {
- "$(CONFIG_COMMAND)" -L"$(LINKPATH)" -Wl,-R$(SPACE)-Wl,"$(RPATH)" -Wl,-rpath-link$(SPACE)-Wl,"$(RPATH_LINK)" -o "$(<)" "$(>)" "$(LIBRARIES)" -l$(FINDLIBS-SA) -l$(FINDLIBS-ST) $(OPTIONS) $(USER_OPTIONS)
-}
-
-rule link.dll ( targets * : sources * : properties * ) {
- gcc.setup-threading $(targets) : $(sources) : $(properties) ;
- gcc.setup-address-model $(targets) : $(sources) : $(properties) ;
- SPACE on $(targets) = " " ;
- JAM_SEMAPHORE on $(targets) = <s>clang-linux-link-semaphore ;
-}
-
-# Differ from 'link' above only by -shared.
-actions link.dll bind LIBRARIES {
- "$(CONFIG_COMMAND)" -L"$(LINKPATH)" -Wl,-R$(SPACE)-Wl,"$(RPATH)" -o "$(<)" -Wl,-soname$(SPACE)-Wl,$(<[1]:D=) -shared "$(>)" "$(LIBRARIES)" -l$(FINDLIBS-SA) -l$(FINDLIBS-ST) $(OPTIONS) $(USER_OPTIONS)
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/clang.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/clang.jam b/ext/kenlm/jam-files/boost-build/tools/clang.jam
deleted file mode 100644
index e0ac9a5..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/clang.jam
+++ /dev/null
@@ -1,27 +0,0 @@
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt
-# or copy at http://www.boost.org/LICENSE_1_0.txt)
-
-# This is a generic 'clang' toolset. Depending on the current system, it
-# forwards either to 'clang-unix' or 'clang-darwin' modules.
-
-import feature ;
-import os ;
-import toolset ;
-
-feature.extend toolset : clang ;
-feature.subfeature toolset clang : platform : : propagated link-incompatible ;
-
-rule init ( * : * )
-{
- if [ os.name ] = MACOSX
- {
- toolset.using clang-darwin :
- $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ;
- }
- else
- {
- toolset.using clang-linux :
- $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/common.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/common.jam b/ext/kenlm/jam-files/boost-build/tools/common.jam
deleted file mode 100644
index 08df2d9..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/common.jam
+++ /dev/null
@@ -1,988 +0,0 @@
-# Copyright 2003, 2005 Dave Abrahams
-# Copyright 2005, 2006 Rene Rivera
-# Copyright 2005 Toon Knapen
-# Copyright 2002, 2003, 2004, 2005, 2006 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-# Provides actions common to all toolsets, such as creating directories and
-# removing files.
-
-import os ;
-import modules ;
-import utility ;
-import print ;
-import type ;
-import feature ;
-import errors ;
-import path ;
-import sequence ;
-import toolset ;
-import virtual-target ;
-
-if [ MATCH (--debug-configuration) : [ modules.peek : ARGV ] ]
-{
- .debug-configuration = true ;
-}
-if [ MATCH (--show-configuration) : [ modules.peek : ARGV ] ]
-{
- .show-configuration = true ;
-}
-
-# Configurations
-#
-# The following class helps to manage toolset configurations. Each configuration
-# has a unique ID and one or more parameters. A typical example of a unique ID
-# is a condition generated by 'common.check-init-parameters' rule. Other kinds
-# of IDs can be used. Parameters may include any details about the configuration
-# like 'command', 'path', etc.
-#
-# A toolset configuration may be in one of the following states:
-#
-# - registered
-# Configuration has been registered (e.g. explicitly or by auto-detection
-# code) but has not yet been marked as used, i.e. 'toolset.using' rule has
-# not yet been called for it.
-# - used
-# Once called 'toolset.using' rule marks the configuration as 'used'.
-#
-# The main difference between the states above is that while a configuration is
-# 'registered' its options can be freely changed. This is useful in particular
-# for autodetection code - all detected configurations may be safely overwritten
-# by user code.
-
-class configurations
-{
- import errors ;
-
- rule __init__ ( )
- {
- }
-
- # Registers a configuration.
- #
- # Returns 'true' if the configuration has been added and an empty value if
- # it already exists. Reports an error if the configuration is 'used'.
- #
- rule register ( id )
- {
- if $(id) in $(self.used)
- {
- errors.error "common: the configuration '$(id)' is in use" ;
- }
-
- local retval ;
-
- if ! $(id) in $(self.all)
- {
- self.all += $(id) ;
-
- # Indicate that a new configuration has been added.
- retval = true ;
- }
-
- return $(retval) ;
- }
-
- # Mark a configuration as 'used'.
- #
- # Returns 'true' if the state of the configuration has been changed to
- # 'used' and an empty value if it the state has not been changed. Reports an
- # error if the configuration is not known.
- #
- rule use ( id )
- {
- if ! $(id) in $(self.all)
- {
- errors.error "common: the configuration '$(id)' is not known" ;
- }
-
- local retval ;
-
- if ! $(id) in $(self.used)
- {
- self.used += $(id) ;
-
- # Indicate that the configuration has been marked as 'used'.
- retval = true ;
- }
-
- return $(retval) ;
- }
-
- # Return all registered configurations.
- #
- rule all ( )
- {
- return $(self.all) ;
- }
-
- # Return all used configurations.
- #
- rule used ( )
- {
- return $(self.used) ;
- }
-
- # Returns the value of a configuration parameter.
- #
- rule get ( id : param )
- {
- return $(self.$(param).$(id)) ;
- }
-
- # Sets the value of a configuration parameter.
- #
- rule set ( id : param : value * )
- {
- self.$(param).$(id) = $(value) ;
- }
-}
-
-
-# The rule for checking toolset parameters. Trailing parameters should all be
-# parameter name/value pairs. The rule will check that each parameter either has
-# a value in each invocation or has no value in each invocation. Also, the rule
-# will check that the combination of all parameter values is unique in all
-# invocations.
-#
-# Each parameter name corresponds to a subfeature. This rule will declare a
-# subfeature the first time a non-empty parameter value is passed and will
-# extend it with all the values.
-#
-# The return value from this rule is a condition to be used for flags settings.
-#
-rule check-init-parameters ( toolset requirement * : * )
-{
- local sig = $(toolset) ;
- local condition = <toolset>$(toolset) ;
- local subcondition ;
- for local index in 2 3 4 5 6 7 8 9
- {
- local name = $($(index)[1]) ;
- local value = $($(index)[2]) ;
-
- if $(value)-is-not-empty
- {
- condition = $(condition)-$(value) ;
- if $(.had-unspecified-value.$(toolset).$(name))
- {
- errors.user-error
- "$(toolset) initialization: parameter '$(name)'"
- "inconsistent" : "no value was specified in earlier"
- "initialization" : "an explicit value is specified now" ;
- }
- # The below logic is for intel compiler. It calls this rule with
- # 'intel-linux' and 'intel-win' as toolset, so we need to get the
- # base part of toolset name. We can not pass 'intel' as toolset
- # because in that case it will be impossible to register versionless
- # intel-linux and intel-win toolsets of a specific version.
- local t = $(toolset) ;
- local m = [ MATCH ([^-]*)- : $(toolset) ] ;
- if $(m)
- {
- t = $(m[1]) ;
- }
- if ! $(.had-value.$(toolset).$(name))
- {
- if ! $(.declared-subfeature.$(t).$(name))
- {
- feature.subfeature toolset $(t) : $(name) : : propagated ;
- .declared-subfeature.$(t).$(name) = true ;
- }
- .had-value.$(toolset).$(name) = true ;
- }
- feature.extend-subfeature toolset $(t) : $(name) : $(value) ;
- subcondition += <toolset-$(t):$(name)>$(value) ;
- }
- else
- {
- if $(.had-value.$(toolset).$(name))
- {
- errors.user-error
- "$(toolset) initialization: parameter '$(name)'"
- "inconsistent" : "an explicit value was specified in an"
- "earlier initialization" : "no value is specified now" ;
- }
- .had-unspecified-value.$(toolset).$(name) = true ;
- }
- sig = $(sig)$(value:E="")- ;
- }
- if $(sig) in $(.all-signatures)
- {
- local message =
- "duplicate initialization of $(toolset) with the following parameters: " ;
- for local index in 2 3 4 5 6 7 8 9
- {
- local p = $($(index)) ;
- if $(p)
- {
- message += "$(p[1]) = $(p[2]:E=<unspecified>)" ;
- }
- }
- message += "previous initialization at $(.init-loc.$(sig))" ;
- errors.user-error
- $(message[1]) : $(message[2]) : $(message[3]) : $(message[4]) :
- $(message[5]) : $(message[6]) : $(message[7]) : $(message[8]) ;
- }
- .all-signatures += $(sig) ;
- .init-loc.$(sig) = [ errors.nearest-user-location ] ;
-
- # If we have a requirement, this version should only be applied under that
- # condition. To accomplish this we add a toolset requirement that imposes
- # the toolset subcondition, which encodes the version.
- if $(requirement)
- {
- local r = <toolset>$(toolset) $(requirement) ;
- r = $(r:J=,) ;
- toolset.add-requirements $(r):$(subcondition) ;
- }
-
- # We add the requirements, if any, to the condition to scope the toolset
- # variables and options to this specific version.
- condition += $(requirement) ;
-
- if $(.show-configuration)
- {
- ECHO notice: $(condition) ;
- }
- return $(condition:J=/) ;
-}
-
-
-# A helper rule to get the command to invoke some tool. If
-# 'user-provided-command' is not given, tries to find binary named 'tool' in
-# PATH and in the passed 'additional-path'. Otherwise, verifies that the first
-# element of 'user-provided-command' is an existing program.
-#
-# This rule returns the command to be used when invoking the tool. If we can not
-# find the tool, a warning is issued. If 'path-last' is specified, PATH is
-# checked after 'additional-paths' when searching for 'tool'.
-#
-rule get-invocation-command-nodefault ( toolset : tool :
- user-provided-command * : additional-paths * : path-last ? )
-{
- local command ;
- if ! $(user-provided-command)
- {
- command = [ find-tool $(tool) : $(additional-paths) : $(path-last) ] ;
- if ! $(command) && $(.debug-configuration)
- {
- ECHO warning: toolset $(toolset) initialization: can not find tool
- $(tool) ;
- ECHO warning: initialized from [ errors.nearest-user-location ] ;
- }
- }
- else
- {
- command = [ check-tool $(user-provided-command) ] ;
- if ! $(command) && $(.debug-configuration)
- {
- ECHO warning: toolset $(toolset) initialization: ;
- ECHO warning: can not find user-provided command
- '$(user-provided-command)' ;
- ECHO warning: initialized from [ errors.nearest-user-location ] ;
- }
- }
-
- return $(command) ;
-}
-
-
-# Same as get-invocation-command-nodefault, except that if no tool is found,
-# returns either the user-provided-command, if present, or the 'tool' parameter.
-#
-rule get-invocation-command ( toolset : tool : user-provided-command * :
- additional-paths * : path-last ? )
-{
- local result = [ get-invocation-command-nodefault $(toolset) : $(tool) :
- $(user-provided-command) : $(additional-paths) : $(path-last) ] ;
-
- if ! $(result)
- {
- if $(user-provided-command)
- {
- result = $(user-provided-command) ;
- }
- else
- {
- result = $(tool) ;
- }
- }
- return $(result) ;
-}
-
-
-# Given an invocation command return the absolute path to the command. This
-# works even if command has no path element and was found on the PATH.
-#
-rule get-absolute-tool-path ( command )
-{
- if $(command:D)
- {
- return $(command:D) ;
- }
- else
- {
- local m = [ GLOB [ modules.peek : PATH Path path ] : $(command)
- $(command).exe ] ;
- return $(m[1]:D) ;
- }
-}
-
-
-# Attempts to find tool (binary) named 'name' in PATH and in 'additional-paths'.
-# If found in PATH, returns 'name' and if found in additional paths, returns
-# absolute name. If the tool is found in several directories, returns the first
-# path found. Otherwise, returns an empty string. If 'path-last' is specified,
-# PATH is searched after 'additional-paths'.
-#
-rule find-tool ( name : additional-paths * : path-last ? )
-{
- local path = [ path.programs-path ] ;
- local match = [ path.glob $(path) : $(name) $(name).exe ] ;
- local additional-match = [ path.glob $(additional-paths) : $(name)
- $(name).exe ] ;
-
- local result ;
- if $(path-last)
- {
- result = $(additional-match) ;
- if ! $(result) && $(match)
- {
- result = $(name) ;
- }
- }
- else
- {
- if $(match)
- {
- result = $(name) ;
- }
- else
- {
- result = $(additional-match) ;
- }
- }
- if $(result)
- {
- return [ path.native $(result[1]) ] ;
- }
-}
-
-# Checks if 'command' can be found either in path or is a full name to an
-# existing file.
-#
-local rule check-tool-aux ( command )
-{
- if $(command:D)
- {
- if [ path.exists $(command) ]
- # Both NT and Cygwin will run .exe files by their unqualified names.
- || ( [ os.on-windows ] && [ path.exists $(command).exe ] )
- # Only NT will run .bat & .cmd files by their unqualified names.
- || ( ( [ os.name ] = NT ) && ( [ path.exists $(command).bat ] ||
- [ path.exists $(command).cmd ] ) )
- {
- return $(command) ;
- }
- }
- else
- {
- if [ GLOB [ modules.peek : PATH Path path ] : $(command) ]
- {
- return $(command) ;
- }
- }
-}
-
-
-# Checks that a tool can be invoked by 'command'. If command is not an absolute
-# path, checks if it can be found in 'path'. If comand is an absolute path,
-# check that it exists. Returns 'command' if ok or empty string otherwise.
-#
-local rule check-tool ( xcommand + )
-{
- if [ check-tool-aux $(xcommand[1]) ] ||
- [ check-tool-aux $(xcommand[-1]) ]
- {
- return $(xcommand) ;
- }
-}
-
-
-# Handle common options for toolset, specifically sets the following flag
-# variables:
-# - CONFIG_COMMAND to $(command)
-# - OPTIONS for compile to the value of <compileflags> in $(options)
-# - OPTIONS for compile.c to the value of <cflags> in $(options)
-# - OPTIONS for compile.c++ to the value of <cxxflags> in $(options)
-# - OPTIONS for compile.fortran to the value of <fflags> in $(options)
-# - OPTIONS for link to the value of <linkflags> in $(options)
-#
-rule handle-options ( toolset : condition * : command * : options * )
-{
- if $(.debug-configuration)
- {
- ECHO notice: will use '$(command)' for $(toolset), condition
- $(condition:E=(empty)) ;
- }
-
- # The last parameter ('unchecked') says it is OK to set flags for another
- # module.
- toolset.flags $(toolset) CONFIG_COMMAND $(condition) : $(command)
- : unchecked ;
-
- toolset.flags $(toolset).compile OPTIONS $(condition) :
- [ feature.get-values <compileflags> : $(options) ] : unchecked ;
-
- toolset.flags $(toolset).compile.c OPTIONS $(condition) :
- [ feature.get-values <cflags> : $(options) ] : unchecked ;
-
- toolset.flags $(toolset).compile.c++ OPTIONS $(condition) :
- [ feature.get-values <cxxflags> : $(options) ] : unchecked ;
-
- toolset.flags $(toolset).compile.fortran OPTIONS $(condition) :
- [ feature.get-values <fflags> : $(options) ] : unchecked ;
-
- toolset.flags $(toolset).link OPTIONS $(condition) :
- [ feature.get-values <linkflags> : $(options) ] : unchecked ;
-}
-
-
-# Returns the location of the "program files" directory on a Windows platform.
-#
-rule get-program-files-dir ( )
-{
- local ProgramFiles = [ modules.peek : ProgramFiles ] ;
- if $(ProgramFiles)
- {
- ProgramFiles = "$(ProgramFiles:J= )" ;
- }
- else
- {
- ProgramFiles = "c:\\Program Files" ;
- }
- return $(ProgramFiles) ;
-}
-
-
-if [ os.name ] = NT
-{
- RM = del /f /q ;
- CP = copy /b ;
- IGNORE = "2>nul >nul & setlocal" ;
- LN ?= $(CP) ;
- # Ugly hack to convince copy to set the timestamp of the destination to the
- # current time by concatenating the source with a nonexistent file. Note
- # that this requires /b (binary) as the default when concatenating files is
- # /a (ascii).
- WINDOWS-CP-HACK = "+ this-file-does-not-exist-A698EE7806899E69" ;
-}
-else
-{
- RM = rm -f ;
- CP = cp ;
- LN = ln ;
-}
-
-
-rule rm-command ( )
-{
- return $(RM) ;
-}
-
-
-rule copy-command ( )
-{
- return $(CP) ;
-}
-
-
-if "\n" = "n"
-{
- # Escape characters not supported so use ugly hacks. Will not work on Cygwin
- # - see below.
- nl = "
-" ;
- q = "" ;
-}
-else
-{
- nl = "\n" ;
- q = "\"" ;
-}
-
-# Returns the command needed to set an environment variable on the current
-# platform. The variable setting persists through all following commands and is
-# visible in the environment seen by subsequently executed commands. In other
-# words, on Unix systems, the variable is exported, which is consistent with the
-# only possible behavior on Windows systems.
-#
-rule variable-setting-command ( variable : value )
-{
- if [ os.name ] = NT
- {
- return "set $(variable)=$(value)$(nl)" ;
- }
- else
- {
- # If we do not have escape character support in bjam, the cod below
- # blows up on CYGWIN, since the $(nl) variable holds a Windows new-line
- # \r\n sequence that messes up the executed export command which then
- # reports that the passed variable name is incorrect.
- # But we have a check for cygwin in kernel/bootstrap.jam already.
- return "$(variable)=$(q)$(value)$(q)$(nl)export $(variable)$(nl)" ;
- }
-}
-
-
-# Returns a command to sets a named shell path variable to the given NATIVE
-# paths on the current platform.
-#
-rule path-variable-setting-command ( variable : paths * )
-{
- local sep = [ os.path-separator ] ;
- return [ variable-setting-command $(variable) : $(paths:J=$(sep)) ] ;
-}
-
-
-# Returns a command that prepends the given paths to the named path variable on
-# the current platform.
-#
-rule prepend-path-variable-command ( variable : paths * )
-{
- return [ path-variable-setting-command $(variable)
- : $(paths) [ os.expand-variable $(variable) ] ] ;
-}
-
-
-# Return a command which can create a file. If 'r' is result of invocation, then
-# 'r foobar' will create foobar with unspecified content. What happens if file
-# already exists is unspecified.
-#
-rule file-creation-command ( )
-{
- if [ os.name ] = NT
- {
- # A few alternative implementations on Windows:
- #
- # 'type NUL >> '
- # That would construct an empty file instead of a file containing
- # a space and an end-of-line marker but it would also not change
- # the target's timestamp in case the file already exists.
- #
- # 'type NUL > '
- # That would construct an empty file instead of a file containing
- # a space and an end-of-line marker but it would also destroy an
- # already existing file by overwriting it with an empty one.
- #
- # I guess the best solution would be to allow Boost Jam to define
- # built-in functions such as 'create a file', 'touch a file' or 'copy a
- # file' which could be used from inside action code. That would allow
- # completely portable operations without this kind of kludge.
- # (22.02.2009.) (Jurko)
- return "echo. > " ;
- }
- else
- {
- return "touch " ;
- }
-}
-
-
-# Returns a command that may be used for 'touching' files. It is not a real
-# 'touch' command on NT because it adds an empty line at the end of file but it
-# works with source files.
-#
-rule file-touch-command ( )
-{
- if [ os.name ] = NT
- {
- return "echo. >> " ;
- }
- else
- {
- return "touch " ;
- }
-}
-
-
-rule MkDir
-{
- # If dir exists, do not update it. Do this even for $(DOT).
- NOUPDATE $(<) ;
-
- if $(<) != $(DOT) && ! $($(<)-mkdir)
- {
- # Cheesy gate to prevent multiple invocations on same dir.
- $(<)-mkdir = true ;
-
- # Schedule the mkdir build action.
- common.mkdir $(<) ;
-
- # Prepare a Jam 'dirs' target that can be used to make the build only
- # construct all the target directories.
- DEPENDS dirs : $(<) ;
-
- # Recursively create parent directories. $(<:P) = $(<)'s parent & we
- # recurse until root.
-
- local s = $(<:P) ;
- if [ os.name ] = NT
- {
- switch $(s)
- {
- case *: : s = ;
- case *:\\ : s = ;
- }
- }
-
- if $(s)
- {
- if $(s) != $(<)
- {
- DEPENDS $(<) : $(s) ;
- MkDir $(s) ;
- }
- else
- {
- NOTFILE $(s) ;
- }
- }
- }
-}
-
-
-#actions MkDir1
-#{
-# mkdir "$(<)"
-#}
-
-# The following quick-fix actions should be replaced using the original MkDir1
-# action once Boost Jam gets updated to correctly detect different paths leading
-# up to the same filesystem target and triggers their build action only once.
-# (todo) (04.07.2008.) (Jurko)
-
-if [ os.name ] = NT
-{
- actions mkdir
- {
- if not exist "$(<)\\" mkdir "$(<)"
- }
-}
-else
-{
- actions mkdir
- {
- mkdir -p "$(<)"
- }
-}
-
-actions piecemeal together existing Clean
-{
- $(RM) "$(>)"
-}
-
-
-rule copy
-{
-}
-
-
-actions copy
-{
- $(CP) "$(>)" $(WINDOWS-CP-HACK) "$(<)"
-}
-
-
-rule RmTemps
-{
-}
-
-
-actions quietly updated piecemeal together RmTemps
-{
- $(RM) "$(>)" $(IGNORE)
-}
-
-
-actions hard-link
-{
- $(RM) "$(<)" 2$(NULL_OUT) $(NULL_OUT)
- $(LN) "$(>)" "$(<)" $(NULL_OUT)
-}
-
-
-# Given a target, as given to a custom tag rule, returns a string formatted
-# according to the passed format. Format is a list of properties that is
-# represented in the result. For each element of format the corresponding target
-# information is obtained and added to the result string. For all, but the
-# literal, the format value is taken as the as string to prepend to the output
-# to join the item to the rest of the result. If not given "-" is used as a
-# joiner.
-#
-# The format options can be:
-#
-# <base>[joiner]
-# :: The basename of the target name.
-# <toolset>[joiner]
-# :: The abbreviated toolset tag being used to build the target.
-# <threading>[joiner]
-# :: Indication of a multi-threaded build.
-# <runtime>[joiner]
-# :: Collective tag of the build runtime.
-# <version:/version-feature | X.Y[.Z]/>[joiner]
-# :: Short version tag taken from the given "version-feature" in the
-# build properties. Or if not present, the literal value as the
-# version number.
-# <property:/property-name/>[joiner]
-# :: Direct lookup of the given property-name value in the build
-# properties. /property-name/ is a regular expression. E.g.
-# <property:toolset-.*:flavor> will match every toolset.
-# /otherwise/
-# :: The literal value of the format argument.
-#
-# For example this format:
-#
-# boost_ <base> <toolset> <threading> <runtime> <version:boost-version>
-#
-# Might return:
-#
-# boost_thread-vc80-mt-gd-1_33.dll, or
-# boost_regex-vc80-gd-1_33.dll
-#
-# The returned name also has the target type specific prefix and suffix which
-# puts it in a ready form to use as the value from a custom tag rule.
-#
-rule format-name ( format * : name : type ? : property-set )
-{
- local result = "" ;
- for local f in $(format)
- {
- switch $(f:G)
- {
- case <base> :
- local matched = [ MATCH "^(boost.*python)-.*" : $(name) ] ;
- if $(matched) = boost_python || $(matched) = boost_mpi_python
- {
- result += $(name) ;
- }
- else
- {
- result += $(name:B) ;
- }
-
- case <toolset> :
- result += [ join-tag $(f:G=) : [ toolset-tag $(name) : $(type) :
- $(property-set) ] ] ;
-
- case <threading> :
- result += [ join-tag $(f:G=) : [ threading-tag $(name) : $(type)
- : $(property-set) ] ] ;
-
- case <runtime> :
- result += [ join-tag $(f:G=) : [ runtime-tag $(name) : $(type) :
- $(property-set) ] ] ;
-
- case <qt> :
- result += [ join-tag $(f:G=) : [ qt-tag $(name) : $(type) :
- $(property-set) ] ] ;
-
- case <address-model> :
- result += [ join-tag $(f:G=) : [ address-model-tag $(name) :
- $(type) : $(property-set) ] ] ;
-
- case <version:*> :
- local key = [ MATCH <version:(.*)> : $(f:G) ] ;
- local version = [ $(property-set).get <$(key)> ] ;
- version ?= $(key) ;
- version = [ MATCH "^([^.]+)[.]([^.]+)[.]?([^.]*)" : $(version) ] ;
- result += [ join-tag $(f:G=) : $(version[1])_$(version[2]) ] ;
-
- case <property:*> :
- local key = [ MATCH <property:(.*)> : $(f:G) ] ;
- local p0 = [ MATCH <($(key))> : [ $(property-set).raw ] ] ;
- if $(p0)
- {
- local p = [ $(property-set).get <$(p0)> ] ;
- if $(p)
- {
- result += [ join-tag $(f:G=) : $(p) ] ;
- }
- }
-
- case * :
- result += $(f:G=) ;
- }
- }
- return [ virtual-target.add-prefix-and-suffix $(result:J=) : $(type) :
- $(property-set) ] ;
-}
-
-
-local rule join-tag ( joiner ? : tag ? )
-{
- if ! $(joiner) { joiner = - ; }
- return $(joiner)$(tag) ;
-}
-
-
-local rule toolset-tag ( name : type ? : property-set )
-{
- local tag = ;
-
- local properties = [ $(property-set).raw ] ;
- switch [ $(property-set).get <toolset> ]
- {
- case borland* : tag += bcb ;
- case clang* :
- {
- switch [ $(property-set).get <toolset-clang:platform> ]
- {
- case darwin : tag += clang-darwin ;
- case linux : tag += clang ;
- }
- }
- case como* : tag += como ;
- case cw : tag += cw ;
- case darwin* : tag += xgcc ;
- case edg* : tag += edg ;
- case gcc* :
- {
- switch [ $(property-set).get <toolset-gcc:flavor> ]
- {
- case *mingw* : tag += mgw ;
- case * : tag += gcc ;
- }
- }
- case intel :
- if [ $(property-set).get <toolset-intel:platform> ] = win
- {
- tag += iw ;
- }
- else
- {
- tag += il ;
- }
- case kcc* : tag += kcc ;
- case kylix* : tag += bck ;
- #case metrowerks* : tag += cw ;
- #case mingw* : tag += mgw ;
- case mipspro* : tag += mp ;
- case msvc* : tag += vc ;
- case qcc* : tag += qcc ;
- case sun* : tag += sw ;
- case tru64cxx* : tag += tru ;
- case vacpp* : tag += xlc ;
- }
- local version = [ MATCH <toolset.*version>([0123456789]+)[.]([0123456789]*)
- : $(properties) ] ;
- # For historical reasons, vc6.0 and vc7.0 use different naming.
- if $(tag) = vc
- {
- if $(version[1]) = 6
- {
- # Cancel minor version.
- version = 6 ;
- }
- else if $(version[1]) = 7 && $(version[2]) = 0
- {
- version = 7 ;
- }
- }
- # On intel, version is not added, because it does not matter and it is the
- # version of vc used as backend that matters. Ideally, we should encode the
- # backend version but that would break compatibility with V1.
- if $(tag) = iw
- {
- version = ;
- }
-
- # On borland, version is not added for compatibility with V1.
- if $(tag) = bcb
- {
- version = ;
- }
-
- tag += $(version) ;
-
- return $(tag:J=) ;
-}
-
-
-local rule threading-tag ( name : type ? : property-set )
-{
- if <threading>multi in [ $(property-set).raw ]
- {
- return mt ;
- }
-}
-
-
-local rule runtime-tag ( name : type ? : property-set )
-{
- local tag = ;
-
- local properties = [ $(property-set).raw ] ;
- if <runtime-link>static in $(properties) { tag += s ; }
-
- # This is an ugly thing. In V1, there is code to automatically detect which
- # properties affect a target. So, if <runtime-debugging> does not affect gcc
- # toolset, the tag rules will not even see <runtime-debugging>. Similar
- # functionality in V2 is not implemented yet, so we just check for toolsets
- # known to care about runtime debugging.
- if ( <toolset>msvc in $(properties) ) ||
- ( <stdlib>stlport in $(properties) ) ||
- ( <toolset-intel:platform>win in $(properties) )
- {
- if <runtime-debugging>on in $(properties) { tag += g ; }
- }
-
- if <python-debugging>on in $(properties) { tag += y ; }
- if <variant>debug in $(properties) { tag += d ; }
- if <stdlib>stlport in $(properties) { tag += p ; }
- if <stdlib-stlport:iostream>hostios in $(properties) { tag += n ; }
-
- return $(tag:J=) ;
-}
-
-
-# Create a tag for the Qt library version
-# "<qt>4.6.0" will result in tag "qt460"
-local rule qt-tag ( name : type ? : property-set )
-{
- local v = [ MATCH ([0123456789]+)[.]?([0123456789]*)[.]?([0123456789]*) :
- [ $(property-set).get <qt> ] ] ;
- return qt$(v:J=) ;
-}
-
-
-# Create a tag for the address-model
-# <address-model>64 will simply generate "64"
-local rule address-model-tag ( name : type ? : property-set )
-{
- return [ $(property-set).get <address-model> ] ;
-}
-
-
-rule __test__ ( )
-{
- import assert ;
-
- local save-os = [ modules.peek os : .name ] ;
-
- modules.poke os : .name : LINUX ;
- assert.result "PATH=\"foo:bar:baz\"\nexport PATH\n"
- : path-variable-setting-command PATH : foo bar baz ;
- assert.result "PATH=\"foo:bar:$PATH\"\nexport PATH\n"
- : prepend-path-variable-command PATH : foo bar ;
-
- modules.poke os : .name : NT ;
- assert.result "set PATH=foo;bar;baz\n"
- : path-variable-setting-command PATH : foo bar baz ;
- assert.result "set PATH=foo;bar;%PATH%\n"
- : prepend-path-variable-command PATH : foo bar ;
-
- modules.poke os : .name : $(save-os) ;
-}
[07/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/vocab.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/vocab.cc b/ext/kenlm/lm/vocab.cc
deleted file mode 100644
index 3d83e04..0000000
--- a/ext/kenlm/lm/vocab.cc
+++ /dev/null
@@ -1,329 +0,0 @@
-#include "lm/vocab.hh"
-
-#include "lm/binary_format.hh"
-#include "lm/enumerate_vocab.hh"
-#include "lm/lm_exception.hh"
-#include "lm/config.hh"
-#include "lm/weights.hh"
-#include "util/exception.hh"
-#include "util/file_stream.hh"
-#include "util/file.hh"
-#include "util/joint_sort.hh"
-#include "util/murmur_hash.hh"
-#include "util/probing_hash_table.hh"
-
-#include <cstring>
-#include <string>
-
-namespace lm {
-namespace ngram {
-
-namespace detail {
-uint64_t HashForVocab(const char *str, std::size_t len) {
- // This proved faster than Boost's hash in speed trials: total load time Murmur 67090000, Boost 72210000
- // Chose to use 64A instead of native so binary format will be portable across 64 and 32 bit.
- return util::MurmurHash64A(str, len, 0);
-}
-} // namespace detail
-
-namespace {
-// Normally static initialization is a bad idea but MurmurHash is pure arithmetic, so this is ok.
-const uint64_t kUnknownHash = detail::HashForVocab("<unk>", 5);
-// Sadly some LMs have <UNK>.
-const uint64_t kUnknownCapHash = detail::HashForVocab("<UNK>", 5);
-
-// TODO: replace with FilePiece.
-void ReadWords(int fd, EnumerateVocab *enumerate, WordIndex expected_count, uint64_t offset) {
- util::SeekOrThrow(fd, offset);
- // Check that we're at the right place by reading <unk> which is always first.
- char check_unk[6];
- util::ReadOrThrow(fd, check_unk, 6);
- UTIL_THROW_IF(
- memcmp(check_unk, "<unk>", 6),
- FormatLoadException,
- "Vocabulary words are in the wrong place. This could be because the binary file was built with stale gcc and old kenlm. Stale gcc, including the gcc distributed with RedHat and OS X, has a bug that ignores pragma pack for template-dependent types. New kenlm works around this, so you'll save memory but have to rebuild any binary files using the probing data structure.");
- if (!enumerate) return;
- enumerate->Add(0, "<unk>");
-
- // Read all the words after unk.
- const std::size_t kInitialRead = 16384;
- std::string buf;
- buf.reserve(kInitialRead + 100);
- buf.resize(kInitialRead);
- WordIndex index = 1; // Read <unk> already.
- while (true) {
- std::size_t got = util::ReadOrEOF(fd, &buf[0], kInitialRead);
- if (got == 0) break;
- buf.resize(got);
- while (buf[buf.size() - 1]) {
- char next_char;
- util::ReadOrThrow(fd, &next_char, 1);
- buf.push_back(next_char);
- }
- // Ok now we have null terminated strings.
- for (const char *i = buf.data(); i != buf.data() + buf.size();) {
- std::size_t length = strlen(i);
- enumerate->Add(index++, StringPiece(i, length));
- i += length + 1 /* null byte */;
- }
- }
-
- UTIL_THROW_IF(expected_count != index, FormatLoadException, "The binary file has the wrong number of words at the end. This could be caused by a truncated binary file.");
-}
-
-// Constructor ordering madness.
-int SeekAndReturn(int fd, uint64_t start) {
- util::SeekOrThrow(fd, start);
- return fd;
-}
-} // namespace
-
-ImmediateWriteWordsWrapper::ImmediateWriteWordsWrapper(EnumerateVocab *inner, int fd, uint64_t start)
- : inner_(inner), stream_(SeekAndReturn(fd, start)) {}
-
-WriteWordsWrapper::WriteWordsWrapper(EnumerateVocab *inner) : inner_(inner) {}
-
-void WriteWordsWrapper::Add(WordIndex index, const StringPiece &str) {
- if (inner_) inner_->Add(index, str);
- buffer_.append(str.data(), str.size());
- buffer_.push_back(0);
-}
-
-void WriteWordsWrapper::Write(int fd, uint64_t start) {
- util::SeekOrThrow(fd, start);
- util::WriteOrThrow(fd, buffer_.data(), buffer_.size());
- // Free memory from the string.
- std::string for_swap;
- std::swap(buffer_, for_swap);
-}
-
-SortedVocabulary::SortedVocabulary() : begin_(NULL), end_(NULL), enumerate_(NULL) {}
-
-uint64_t SortedVocabulary::Size(uint64_t entries, const Config &/*config*/) {
- // Lead with the number of entries.
- return sizeof(uint64_t) + sizeof(uint64_t) * entries;
-}
-
-void SortedVocabulary::SetupMemory(void *start, std::size_t allocated, std::size_t entries, const Config &config) {
- assert(allocated >= Size(entries, config));
- // Leave space for number of entries.
- begin_ = reinterpret_cast<uint64_t*>(start) + 1;
- end_ = begin_;
- saw_unk_ = false;
-}
-
-void SortedVocabulary::Relocate(void *new_start) {
- std::size_t delta = end_ - begin_;
- begin_ = reinterpret_cast<uint64_t*>(new_start) + 1;
- end_ = begin_ + delta;
-}
-
-void SortedVocabulary::ConfigureEnumerate(EnumerateVocab *to, std::size_t max_entries) {
- enumerate_ = to;
- if (enumerate_) {
- enumerate_->Add(0, "<unk>");
- strings_to_enumerate_.resize(max_entries);
- }
-}
-
-WordIndex SortedVocabulary::Insert(const StringPiece &str) {
- uint64_t hashed = detail::HashForVocab(str);
- if (hashed == kUnknownHash || hashed == kUnknownCapHash) {
- saw_unk_ = true;
- return 0;
- }
- *end_ = hashed;
- if (enumerate_) {
- void *copied = string_backing_.Allocate(str.size());
- memcpy(copied, str.data(), str.size());
- strings_to_enumerate_[end_ - begin_] = StringPiece(static_cast<const char*>(copied), str.size());
- }
- ++end_;
- // This is 1 + the offset where it was inserted to make room for unk.
- return end_ - begin_;
-}
-
-void SortedVocabulary::FinishedLoading(ProbBackoff *reorder) {
- GenericFinished(reorder);
-}
-
-namespace {
-#pragma pack(push)
-#pragma pack(4)
-struct RenumberEntry {
- uint64_t hash;
- const char *str;
- WordIndex old;
- bool operator<(const RenumberEntry &other) const {
- return hash < other.hash;
- }
-};
-#pragma pack(pop)
-} // namespace
-
-void SortedVocabulary::ComputeRenumbering(WordIndex types, int from_words, int to_words, std::vector<WordIndex> &mapping) {
- mapping.clear();
- uint64_t file_size = util::SizeOrThrow(from_words);
- util::scoped_memory strings;
- util::MapRead(util::POPULATE_OR_READ, from_words, 0, file_size, strings);
- const char *const start = static_cast<const char*>(strings.get());
- UTIL_THROW_IF(memcmp(start, "<unk>", 6), FormatLoadException, "Vocab file does not begin with <unk> followed by null");
- std::vector<RenumberEntry> entries;
- entries.reserve(types - 1);
- RenumberEntry entry;
- entry.old = 1;
- for (entry.str = start + 6 /* skip <unk>\0 */; entry.str < start + file_size; ++entry.old) {
- StringPiece str(entry.str, strlen(entry.str));
- entry.hash = detail::HashForVocab(str);
- entries.push_back(entry);
- entry.str += str.size() + 1;
- }
- UTIL_THROW_IF2(entries.size() != types - 1, "Wrong number of vocab ids. Got " << (entries.size() + 1) << " expected " << types);
- std::sort(entries.begin(), entries.end());
- // Write out new vocab file.
- {
- util::FileStream out(to_words);
- out << "<unk>" << '\0';
- for (std::vector<RenumberEntry>::const_iterator i = entries.begin(); i != entries.end(); ++i) {
- out << i->str << '\0';
- }
- }
- strings.reset();
-
- mapping.resize(types);
- mapping[0] = 0; // <unk>
- for (std::vector<RenumberEntry>::const_iterator i = entries.begin(); i != entries.end(); ++i) {
- mapping[i->old] = i + 1 - entries.begin();
- }
-}
-
-void SortedVocabulary::Populated() {
- saw_unk_ = true;
- SetSpecial(Index("<s>"), Index("</s>"), 0);
- bound_ = end_ - begin_ + 1;
- *(reinterpret_cast<uint64_t*>(begin_) - 1) = end_ - begin_;
-}
-
-void SortedVocabulary::LoadedBinary(bool have_words, int fd, EnumerateVocab *to, uint64_t offset) {
- end_ = begin_ + *(reinterpret_cast<const uint64_t*>(begin_) - 1);
- SetSpecial(Index("<s>"), Index("</s>"), 0);
- bound_ = end_ - begin_ + 1;
- if (have_words) ReadWords(fd, to, bound_, offset);
-}
-
-template <class T> void SortedVocabulary::GenericFinished(T *reorder) {
- if (enumerate_) {
- if (!strings_to_enumerate_.empty()) {
- util::PairedIterator<T*, StringPiece*> values(reorder + 1, &*strings_to_enumerate_.begin());
- util::JointSort(begin_, end_, values);
- }
- for (WordIndex i = 0; i < static_cast<WordIndex>(end_ - begin_); ++i) {
- // <unk> strikes again: +1 here.
- enumerate_->Add(i + 1, strings_to_enumerate_[i]);
- }
- strings_to_enumerate_.clear();
- string_backing_.FreeAll();
- } else {
- util::JointSort(begin_, end_, reorder + 1);
- }
- SetSpecial(Index("<s>"), Index("</s>"), 0);
- // Save size. Excludes UNK.
- *(reinterpret_cast<uint64_t*>(begin_) - 1) = end_ - begin_;
- // Includes UNK.
- bound_ = end_ - begin_ + 1;
-}
-
-namespace {
-const unsigned int kProbingVocabularyVersion = 0;
-} // namespace
-
-namespace detail {
-struct ProbingVocabularyHeader {
- // Lowest unused vocab id. This is also the number of words, including <unk>.
- unsigned int version;
- WordIndex bound;
-};
-} // namespace detail
-
-ProbingVocabulary::ProbingVocabulary() : enumerate_(NULL) {}
-
-uint64_t ProbingVocabulary::Size(uint64_t entries, float probing_multiplier) {
- return ALIGN8(sizeof(detail::ProbingVocabularyHeader)) + Lookup::Size(entries, probing_multiplier);
-}
-
-uint64_t ProbingVocabulary::Size(uint64_t entries, const Config &config) {
- return Size(entries, config.probing_multiplier);
-}
-
-void ProbingVocabulary::SetupMemory(void *start, std::size_t allocated) {
- header_ = static_cast<detail::ProbingVocabularyHeader*>(start);
- lookup_ = Lookup(static_cast<uint8_t*>(start) + ALIGN8(sizeof(detail::ProbingVocabularyHeader)), allocated);
- bound_ = 1;
- saw_unk_ = false;
-}
-
-void ProbingVocabulary::Relocate(void *new_start) {
- header_ = static_cast<detail::ProbingVocabularyHeader*>(new_start);
- lookup_.Relocate(static_cast<uint8_t*>(new_start) + ALIGN8(sizeof(detail::ProbingVocabularyHeader)));
-}
-
-void ProbingVocabulary::ConfigureEnumerate(EnumerateVocab *to, std::size_t /*max_entries*/) {
- enumerate_ = to;
- if (enumerate_) {
- enumerate_->Add(0, "<unk>");
- }
-}
-
-WordIndex ProbingVocabulary::Insert(const StringPiece &str) {
- uint64_t hashed = detail::HashForVocab(str);
- // Prevent unknown from going into the table.
- if (hashed == kUnknownHash || hashed == kUnknownCapHash) {
- saw_unk_ = true;
- return 0;
- } else {
- if (enumerate_) enumerate_->Add(bound_, str);
- lookup_.Insert(ProbingVocabularyEntry::Make(hashed, bound_));
- return bound_++;
- }
-}
-
-void ProbingVocabulary::InternalFinishedLoading() {
- lookup_.FinishedInserting();
- header_->bound = bound_;
- header_->version = kProbingVocabularyVersion;
- SetSpecial(Index("<s>"), Index("</s>"), 0);
-}
-
-void ProbingVocabulary::LoadedBinary(bool have_words, int fd, EnumerateVocab *to, uint64_t offset) {
- UTIL_THROW_IF(header_->version != kProbingVocabularyVersion, FormatLoadException, "The binary file has probing version " << header_->version << " but the code expects version " << kProbingVocabularyVersion << ". Please rerun build_binary using the same version of the code.");
- bound_ = header_->bound;
- SetSpecial(Index("<s>"), Index("</s>"), 0);
- if (have_words) ReadWords(fd, to, bound_, offset);
-}
-
-void MissingUnknown(const Config &config) throw(SpecialWordMissingException) {
- switch(config.unknown_missing) {
- case SILENT:
- return;
- case COMPLAIN:
- if (config.messages) *config.messages << "The ARPA file is missing <unk>. Substituting log10 probability " << config.unknown_missing_logprob << "." << std::endl;
- break;
- case THROW_UP:
- UTIL_THROW(SpecialWordMissingException, "The ARPA file is missing <unk> and the model is configured to throw an exception.");
- }
-}
-
-void MissingSentenceMarker(const Config &config, const char *str) throw(SpecialWordMissingException) {
- switch (config.sentence_marker_missing) {
- case SILENT:
- return;
- case COMPLAIN:
- if (config.messages) *config.messages << "Missing special word " << str << "; will treat it as <unk>.";
- break;
- case THROW_UP:
- UTIL_THROW(SpecialWordMissingException, "The ARPA file is missing " << str << " and the model is configured to reject these models. Run build_binary -s to disable this check.");
- }
-}
-
-} // namespace ngram
-} // namespace lm
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/vocab.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/vocab.hh b/ext/kenlm/lm/vocab.hh
deleted file mode 100644
index 59740e8..0000000
--- a/ext/kenlm/lm/vocab.hh
+++ /dev/null
@@ -1,279 +0,0 @@
-#ifndef LM_VOCAB_H
-#define LM_VOCAB_H
-
-#include "lm/enumerate_vocab.hh"
-#include "lm/lm_exception.hh"
-#include "lm/virtual_interface.hh"
-#include "util/file_stream.hh"
-#include "util/murmur_hash.hh"
-#include "util/pool.hh"
-#include "util/probing_hash_table.hh"
-#include "util/sorted_uniform.hh"
-#include "util/string_piece.hh"
-
-#include <limits>
-#include <string>
-#include <vector>
-
-namespace lm {
-struct ProbBackoff;
-class EnumerateVocab;
-
-namespace ngram {
-struct Config;
-
-namespace detail {
-uint64_t HashForVocab(const char *str, std::size_t len);
-inline uint64_t HashForVocab(const StringPiece &str) {
- return HashForVocab(str.data(), str.length());
-}
-struct ProbingVocabularyHeader;
-} // namespace detail
-
-// Writes words immediately to a file instead of buffering, because we know
-// where in the file to put them.
-class ImmediateWriteWordsWrapper : public EnumerateVocab {
- public:
- ImmediateWriteWordsWrapper(EnumerateVocab *inner, int fd, uint64_t start);
-
- void Add(WordIndex index, const StringPiece &str) {
- stream_ << str << '\0';
- if (inner_) inner_->Add(index, str);
- }
-
- private:
- EnumerateVocab *inner_;
-
- util::FileStream stream_;
-};
-
-// When the binary size isn't known yet.
-class WriteWordsWrapper : public EnumerateVocab {
- public:
- WriteWordsWrapper(EnumerateVocab *inner);
-
- void Add(WordIndex index, const StringPiece &str);
-
- const std::string &Buffer() const { return buffer_; }
- void Write(int fd, uint64_t start);
-
- private:
- EnumerateVocab *inner_;
-
- std::string buffer_;
-};
-
-// Vocabulary based on sorted uniform find storing only uint64_t values and using their offsets as indices.
-class SortedVocabulary : public base::Vocabulary {
- public:
- SortedVocabulary();
-
- WordIndex Index(const StringPiece &str) const {
- const uint64_t *found;
- if (util::BoundedSortedUniformFind<const uint64_t*, util::IdentityAccessor<uint64_t>, util::Pivot64>(
- util::IdentityAccessor<uint64_t>(),
- begin_ - 1, 0,
- end_, std::numeric_limits<uint64_t>::max(),
- detail::HashForVocab(str), found)) {
- return found - begin_ + 1; // +1 because <unk> is 0 and does not appear in the lookup table.
- } else {
- return 0;
- }
- }
-
- // Size for purposes of file writing
- static uint64_t Size(uint64_t entries, const Config &config);
-
- /* Read null-delimited words from file from_words, renumber according to
- * hash order, write null-delimited words to to_words, and create a mapping
- * from old id to new id. The 0th vocab word must be <unk>.
- */
- static void ComputeRenumbering(WordIndex types, int from_words, int to_words, std::vector<WordIndex> &mapping);
-
- // Vocab words are [0, Bound()) Only valid after FinishedLoading/LoadedBinary.
- WordIndex Bound() const { return bound_; }
-
- // Everything else is for populating. I'm too lazy to hide and friend these, but you'll only get a const reference anyway.
- void SetupMemory(void *start, std::size_t allocated, std::size_t entries, const Config &config);
-
- void Relocate(void *new_start);
-
- void ConfigureEnumerate(EnumerateVocab *to, std::size_t max_entries);
-
- // Insert and FinishedLoading go together.
- WordIndex Insert(const StringPiece &str);
- // Reorders reorder_vocab so that the IDs are sorted.
- void FinishedLoading(ProbBackoff *reorder_vocab);
-
- // Trie stores the correct counts including <unk> in the header. If this was previously sized based on a count exluding <unk>, padding with 8 bytes will make it the correct size based on a count including <unk>.
- std::size_t UnkCountChangePadding() const { return SawUnk() ? 0 : sizeof(uint64_t); }
-
- bool SawUnk() const { return saw_unk_; }
-
- void LoadedBinary(bool have_words, int fd, EnumerateVocab *to, uint64_t offset);
-
- uint64_t *&EndHack() { return end_; }
-
- void Populated();
-
- private:
- template <class T> void GenericFinished(T *reorder);
-
- uint64_t *begin_, *end_;
-
- WordIndex bound_;
-
- bool saw_unk_;
-
- EnumerateVocab *enumerate_;
-
- // Actual strings. Used only when loading from ARPA and enumerate_ != NULL
- util::Pool string_backing_;
-
- std::vector<StringPiece> strings_to_enumerate_;
-};
-
-#pragma pack(push)
-#pragma pack(4)
-struct ProbingVocabularyEntry {
- uint64_t key;
- WordIndex value;
-
- typedef uint64_t Key;
- uint64_t GetKey() const { return key; }
- void SetKey(uint64_t to) { key = to; }
-
- static ProbingVocabularyEntry Make(uint64_t key, WordIndex value) {
- ProbingVocabularyEntry ret;
- ret.key = key;
- ret.value = value;
- return ret;
- }
-};
-#pragma pack(pop)
-
-// Vocabulary storing a map from uint64_t to WordIndex.
-class ProbingVocabulary : public base::Vocabulary {
- public:
- ProbingVocabulary();
-
- WordIndex Index(const StringPiece &str) const {
- Lookup::ConstIterator i;
- return lookup_.Find(detail::HashForVocab(str), i) ? i->value : 0;
- }
-
- static uint64_t Size(uint64_t entries, float probing_multiplier);
- // This just unwraps Config to get the probing_multiplier.
- static uint64_t Size(uint64_t entries, const Config &config);
-
- // Vocab words are [0, Bound()).
- WordIndex Bound() const { return bound_; }
-
- // Everything else is for populating. I'm too lazy to hide and friend these, but you'll only get a const reference anyway.
- void SetupMemory(void *start, std::size_t allocated);
- void SetupMemory(void *start, std::size_t allocated, std::size_t /*entries*/, const Config &/*config*/) {
- SetupMemory(start, allocated);
- }
-
- void Relocate(void *new_start);
-
- void ConfigureEnumerate(EnumerateVocab *to, std::size_t max_entries);
-
- WordIndex Insert(const StringPiece &str);
-
- template <class Weights> void FinishedLoading(Weights * /*reorder_vocab*/) {
- InternalFinishedLoading();
- }
-
- std::size_t UnkCountChangePadding() const { return 0; }
-
- bool SawUnk() const { return saw_unk_; }
-
- void LoadedBinary(bool have_words, int fd, EnumerateVocab *to, uint64_t offset);
-
- private:
- void InternalFinishedLoading();
-
- typedef util::ProbingHashTable<ProbingVocabularyEntry, util::IdentityHash> Lookup;
-
- Lookup lookup_;
-
- WordIndex bound_;
-
- bool saw_unk_;
-
- EnumerateVocab *enumerate_;
-
- detail::ProbingVocabularyHeader *header_;
-};
-
-void MissingUnknown(const Config &config) throw(SpecialWordMissingException);
-void MissingSentenceMarker(const Config &config, const char *str) throw(SpecialWordMissingException);
-
-template <class Vocab> void CheckSpecials(const Config &config, const Vocab &vocab) throw(SpecialWordMissingException) {
- if (!vocab.SawUnk()) MissingUnknown(config);
- if (vocab.BeginSentence() == vocab.NotFound()) MissingSentenceMarker(config, "<s>");
- if (vocab.EndSentence() == vocab.NotFound()) MissingSentenceMarker(config, "</s>");
-}
-
-class WriteUniqueWords {
- public:
- explicit WriteUniqueWords(int fd) : word_list_(fd) {}
-
- void operator()(const StringPiece &word) {
- word_list_ << word << '\0';
- }
-
- private:
- util::FileStream word_list_;
-};
-
-class NoOpUniqueWords {
- public:
- NoOpUniqueWords() {}
- void operator()(const StringPiece &word) {}
-};
-
-template <class NewWordAction = NoOpUniqueWords> class GrowableVocab {
- public:
- static std::size_t MemUsage(WordIndex content) {
- return Lookup::MemUsage(content > 2 ? content : 2);
- }
-
- // Does not take ownership of write_wordi
- template <class NewWordConstruct> GrowableVocab(WordIndex initial_size, const NewWordConstruct &new_word_construct = NewWordAction())
- : lookup_(initial_size), new_word_(new_word_construct) {
- FindOrInsert("<unk>"); // Force 0
- FindOrInsert("<s>"); // Force 1
- FindOrInsert("</s>"); // Force 2
- }
-
- WordIndex Index(const StringPiece &str) const {
- Lookup::ConstIterator i;
- return lookup_.Find(detail::HashForVocab(str), i) ? i->value : 0;
- }
-
- WordIndex FindOrInsert(const StringPiece &word) {
- ProbingVocabularyEntry entry = ProbingVocabularyEntry::Make(util::MurmurHashNative(word.data(), word.size()), Size());
- Lookup::MutableIterator it;
- if (!lookup_.FindOrInsert(entry, it)) {
- new_word_(word);
- UTIL_THROW_IF(Size() >= std::numeric_limits<lm::WordIndex>::max(), VocabLoadException, "Too many vocabulary words. Change WordIndex to uint64_t in lm/word_index.hh");
- }
- return it->value;
- }
-
- WordIndex Size() const { return lookup_.Size(); }
-
- private:
- typedef util::AutoProbing<ProbingVocabularyEntry, util::IdentityHash> Lookup;
-
- Lookup lookup_;
-
- NewWordAction new_word_;
-};
-
-} // namespace ngram
-} // namespace lm
-
-#endif // LM_VOCAB_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/weights.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/weights.hh b/ext/kenlm/lm/weights.hh
deleted file mode 100644
index f143127..0000000
--- a/ext/kenlm/lm/weights.hh
+++ /dev/null
@@ -1,22 +0,0 @@
-#ifndef LM_WEIGHTS_H
-#define LM_WEIGHTS_H
-
-// Weights for n-grams. Probability and possibly a backoff.
-
-namespace lm {
-struct Prob {
- float prob;
-};
-// No inheritance so this will be a POD.
-struct ProbBackoff {
- float prob;
- float backoff;
-};
-struct RestWeights {
- float prob;
- float backoff;
- float rest;
-};
-
-} // namespace lm
-#endif // LM_WEIGHTS_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/word_index.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/word_index.hh b/ext/kenlm/lm/word_index.hh
deleted file mode 100644
index 59b24d7..0000000
--- a/ext/kenlm/lm/word_index.hh
+++ /dev/null
@@ -1,15 +0,0 @@
-// Separate header because this is used often.
-#ifndef LM_WORD_INDEX_H
-#define LM_WORD_INDEX_H
-
-#include <climits>
-
-namespace lm {
-typedef unsigned int WordIndex;
-const WordIndex kMaxWordIndex = UINT_MAX;
-const WordIndex kUNK = 0;
-} // namespace lm
-
-typedef lm::WordIndex LMWordIndex;
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/wrappers/README
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/wrappers/README b/ext/kenlm/lm/wrappers/README
deleted file mode 100644
index 56c34c2..0000000
--- a/ext/kenlm/lm/wrappers/README
+++ /dev/null
@@ -1,3 +0,0 @@
-This directory is for wrappers around other people's LMs, presenting an interface similar to KenLM's. You will need to have their LM installed.
-
-NPLM is a work in progress.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/wrappers/nplm.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/wrappers/nplm.cc b/ext/kenlm/lm/wrappers/nplm.cc
deleted file mode 100644
index 9bd7c1e..0000000
--- a/ext/kenlm/lm/wrappers/nplm.cc
+++ /dev/null
@@ -1,116 +0,0 @@
-#include "lm/wrappers/nplm.hh"
-#include "util/exception.hh"
-#include "util/file.hh"
-
-#include <algorithm>
-#include <cstring>
-
-#include "neuralLM.h"
-
-namespace lm {
-namespace np {
-
-Vocabulary::Vocabulary(const nplm::vocabulary &vocab)
- : base::Vocabulary(vocab.lookup_word("<s>"), vocab.lookup_word("</s>"), vocab.lookup_word("<unk>")),
- vocab_(vocab), null_word_(vocab.lookup_word("<null>")) {}
-
-Vocabulary::~Vocabulary() {}
-
-WordIndex Vocabulary::Index(const std::string &str) const {
- return vocab_.lookup_word(str);
-}
-
-class Backend {
- public:
- Backend(const nplm::neuralLM &from, const std::size_t cache_size) : lm_(from), ngram_(from.get_order()) {
- lm_.set_cache(cache_size);
- }
-
- nplm::neuralLM &LM() { return lm_; }
- const nplm::neuralLM &LM() const { return lm_; }
-
- Eigen::Matrix<int,Eigen::Dynamic,1> &staging_ngram() { return ngram_; }
-
- double lookup_from_staging() { return lm_.lookup_ngram(ngram_); }
-
- int order() const { return lm_.get_order(); }
-
- private:
- nplm::neuralLM lm_;
- Eigen::Matrix<int,Eigen::Dynamic,1> ngram_;
-};
-
-bool Model::Recognize(const std::string &name) {
- try {
- util::scoped_fd file(util::OpenReadOrThrow(name.c_str()));
- char magic_check[16];
- util::ReadOrThrow(file.get(), magic_check, sizeof(magic_check));
- const char nnlm_magic[] = "\\config\nversion ";
- return !memcmp(magic_check, nnlm_magic, 16);
- } catch (const util::Exception &) {
- return false;
- }
-}
-
-namespace {
-nplm::neuralLM *LoadNPLM(const std::string &file) {
- util::scoped_ptr<nplm::neuralLM> ret(new nplm::neuralLM());
- ret->read(file);
- return ret.release();
-}
-} // namespace
-
-Model::Model(const std::string &file, std::size_t cache)
- : base_instance_(LoadNPLM(file)), vocab_(base_instance_->get_vocabulary()), cache_size_(cache) {
- UTIL_THROW_IF(base_instance_->get_order() > NPLM_MAX_ORDER, util::Exception, "This NPLM has order " << (unsigned int)base_instance_->get_order() << " but the KenLM wrapper was compiled with " << NPLM_MAX_ORDER << ". Change the defintion of NPLM_MAX_ORDER and recompile.");
- // log10 compatible with backoff models.
- base_instance_->set_log_base(10.0);
- State begin_sentence, null_context;
- std::fill(begin_sentence.words, begin_sentence.words + NPLM_MAX_ORDER - 1, base_instance_->lookup_word("<s>"));
- null_word_ = base_instance_->lookup_word("<null>");
- std::fill(null_context.words, null_context.words + NPLM_MAX_ORDER - 1, null_word_);
-
- Init(begin_sentence, null_context, vocab_, base_instance_->get_order());
-}
-
-Model::~Model() {}
-
-FullScoreReturn Model::FullScore(const State &from, const WordIndex new_word, State &out_state) const {
- Backend *backend = backend_.get();
- if (!backend) {
- backend = new Backend(*base_instance_, cache_size_);
- backend_.reset(backend);
- }
- // State is in natural word order.
- FullScoreReturn ret;
- for (int i = 0; i < backend->order() - 1; ++i) {
- backend->staging_ngram()(i) = from.words[i];
- }
- backend->staging_ngram()(backend->order() - 1) = new_word;
- ret.prob = backend->lookup_from_staging();
- // Always say full order.
- ret.ngram_length = backend->order();
- // Shift everything down by one.
- memcpy(out_state.words, from.words + 1, sizeof(WordIndex) * (backend->order() - 2));
- out_state.words[backend->order() - 2] = new_word;
- // Fill in trailing words with zeros so state comparison works.
- memset(out_state.words + backend->order() - 1, 0, sizeof(WordIndex) * (NPLM_MAX_ORDER - backend->order()));
- return ret;
-}
-
-// TODO: optimize with direct call?
-FullScoreReturn Model::FullScoreForgotState(const WordIndex *context_rbegin, const WordIndex *context_rend, const WordIndex new_word, State &out_state) const {
- // State is in natural word order. The API here specifies reverse order.
- std::size_t state_length = std::min<std::size_t>(Order() - 1, context_rend - context_rbegin);
- State state;
- // Pad with null words.
- for (lm::WordIndex *i = state.words; i < state.words + Order() - 1 - state_length; ++i) {
- *i = null_word_;
- }
- // Put new words at the end.
- std::reverse_copy(context_rbegin, context_rbegin + state_length, state.words + Order() - 1 - state_length);
- return FullScore(state, new_word, out_state);
-}
-
-} // namespace np
-} // namespace lm
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/wrappers/nplm.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/wrappers/nplm.hh b/ext/kenlm/lm/wrappers/nplm.hh
deleted file mode 100644
index 82b38fd..0000000
--- a/ext/kenlm/lm/wrappers/nplm.hh
+++ /dev/null
@@ -1,85 +0,0 @@
-#ifndef LM_WRAPPERS_NPLM_H
-#define LM_WRAPPERS_NPLM_H
-
-#include "lm/facade.hh"
-#include "lm/max_order.hh"
-#include "util/string_piece.hh"
-
-#include <boost/thread/tss.hpp>
-#include <boost/scoped_ptr.hpp>
-
-/* Wrapper to NPLM "by Ashish Vaswani, with contributions from David Chiang
- * and Victoria Fossum."
- * http://nlg.isi.edu/software/nplm/
- */
-
-namespace nplm {
-class vocabulary;
-class neuralLM;
-} // namespace nplm
-
-namespace lm {
-namespace np {
-
-class Vocabulary : public base::Vocabulary {
- public:
- Vocabulary(const nplm::vocabulary &vocab);
-
- ~Vocabulary();
-
- WordIndex Index(const std::string &str) const;
-
- // TODO: lobby them to support StringPiece
- WordIndex Index(const StringPiece &str) const {
- return Index(std::string(str.data(), str.size()));
- }
-
- lm::WordIndex NullWord() const { return null_word_; }
-
- private:
- const nplm::vocabulary &vocab_;
-
- const lm::WordIndex null_word_;
-};
-
-// Sorry for imposing my limitations on your code.
-#define NPLM_MAX_ORDER 7
-
-struct State {
- WordIndex words[NPLM_MAX_ORDER - 1];
-};
-
-class Backend;
-
-class Model : public lm::base::ModelFacade<Model, State, Vocabulary> {
- private:
- typedef lm::base::ModelFacade<Model, State, Vocabulary> P;
-
- public:
- // Does this look like an NPLM?
- static bool Recognize(const std::string &file);
-
- explicit Model(const std::string &file, std::size_t cache_size = 1 << 20);
-
- ~Model();
-
- FullScoreReturn FullScore(const State &from, const WordIndex new_word, State &out_state) const;
-
- FullScoreReturn FullScoreForgotState(const WordIndex *context_rbegin, const WordIndex *context_rend, const WordIndex new_word, State &out_state) const;
-
- private:
- boost::scoped_ptr<nplm::neuralLM> base_instance_;
-
- mutable boost::thread_specific_ptr<Backend> backend_;
-
- Vocabulary vocab_;
-
- lm::WordIndex null_word_;
-
- const std::size_t cache_size_;
-};
-
-} // namespace np
-} // namespace lm
-
-#endif // LM_WRAPPERS_NPLM_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/python/_kenlm.pxd
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/python/_kenlm.pxd b/ext/kenlm/python/_kenlm.pxd
deleted file mode 100644
index e0c0248..0000000
--- a/ext/kenlm/python/_kenlm.pxd
+++ /dev/null
@@ -1,33 +0,0 @@
-cdef extern from "lm/word_index.hh" namespace "lm":
- ctypedef unsigned WordIndex
-
-cdef extern from "lm/return.hh" namespace "lm":
- cdef struct FullScoreReturn:
- float prob
- unsigned char ngram_length
-
-cdef extern from "lm/state.hh" namespace "lm::ngram":
- cdef cppclass State :
- int Compare(const State &other) const
-
- int hash_value(const State &state)
-
-cdef extern from "lm/virtual_interface.hh" namespace "lm::base":
- cdef cppclass Vocabulary:
- WordIndex Index(char*)
- WordIndex BeginSentence()
- WordIndex EndSentence()
- WordIndex NotFound()
-
- ctypedef Vocabulary const_Vocabulary "const lm::base::Vocabulary"
-
- cdef cppclass Model:
- void BeginSentenceWrite(void *)
- void NullContextWrite(void *)
- unsigned int Order()
- const_Vocabulary& BaseVocabulary()
- float BaseScore(void *in_state, WordIndex new_word, void *out_state)
- FullScoreReturn BaseFullScore(void *in_state, WordIndex new_word, void *out_state)
-
-cdef extern from "lm/model.hh" namespace "lm::ngram":
- cdef Model *LoadVirtual(char *) except +
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/python/example.py
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/python/example.py b/ext/kenlm/python/example.py
deleted file mode 100644
index 8a18f3a..0000000
--- a/ext/kenlm/python/example.py
+++ /dev/null
@@ -1,28 +0,0 @@
-import os
-import kenlm
-
-LM = os.path.join(os.path.dirname(__file__), '..', 'lm', 'test.arpa')
-model = kenlm.LanguageModel(LM)
-print('{0}-gram model'.format(model.order))
-
-sentence = 'language modeling is fun .'
-print(sentence)
-print(model.score(sentence))
-
-# Check that total full score = direct score
-def score(s):
- return sum(prob for prob, _, _ in model.full_scores(s))
-
-assert (abs(score(sentence) - model.score(sentence)) < 1e-3)
-
-# Show scores and n-gram matches
-words = ['<s>'] + sentence.split() + ['</s>']
-for i, (prob, length, oov) in enumerate(model.full_scores(sentence)):
- print('{0} {1}: {2}'.format(prob, length, ' '.join(words[i+2-length:i+2])))
- if oov:
- print('\t"{0}" is an OOV'.format(words[i+1]))
-
-# Find out-of-vocabulary words
-for w in words:
- if not w in model:
- print('"{0}" is an OOV'.format(w))
[04/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/double-conversion/bignum.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/double-conversion/bignum.h b/ext/kenlm/util/double-conversion/bignum.h
deleted file mode 100644
index 5ec3544..0000000
--- a/ext/kenlm/util/double-conversion/bignum.h
+++ /dev/null
@@ -1,145 +0,0 @@
-// Copyright 2010 the V8 project authors. All rights reserved.
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following
-// disclaimer in the documentation and/or other materials provided
-// with the distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived
-// from this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#ifndef DOUBLE_CONVERSION_BIGNUM_H_
-#define DOUBLE_CONVERSION_BIGNUM_H_
-
-#include "utils.h"
-
-namespace double_conversion {
-
-class Bignum {
- public:
- // 3584 = 128 * 28. We can represent 2^3584 > 10^1000 accurately.
- // This bignum can encode much bigger numbers, since it contains an
- // exponent.
- static const int kMaxSignificantBits = 3584;
-
- Bignum();
- void AssignUInt16(uint16_t value);
- void AssignUInt64(uint64_t value);
- void AssignBignum(const Bignum& other);
-
- void AssignDecimalString(Vector<const char> value);
- void AssignHexString(Vector<const char> value);
-
- void AssignPowerUInt16(uint16_t base, int exponent);
-
- void AddUInt16(uint16_t operand);
- void AddUInt64(uint64_t operand);
- void AddBignum(const Bignum& other);
- // Precondition: this >= other.
- void SubtractBignum(const Bignum& other);
-
- void Square();
- void ShiftLeft(int shift_amount);
- void MultiplyByUInt32(uint32_t factor);
- void MultiplyByUInt64(uint64_t factor);
- void MultiplyByPowerOfTen(int exponent);
- void Times10() { return MultiplyByUInt32(10); }
- // Pseudocode:
- // int result = this / other;
- // this = this % other;
- // In the worst case this function is in O(this/other).
- uint16_t DivideModuloIntBignum(const Bignum& other);
-
- bool ToHexString(char* buffer, int buffer_size) const;
-
- // Returns
- // -1 if a < b,
- // 0 if a == b, and
- // +1 if a > b.
- static int Compare(const Bignum& a, const Bignum& b);
- static bool Equal(const Bignum& a, const Bignum& b) {
- return Compare(a, b) == 0;
- }
- static bool LessEqual(const Bignum& a, const Bignum& b) {
- return Compare(a, b) <= 0;
- }
- static bool Less(const Bignum& a, const Bignum& b) {
- return Compare(a, b) < 0;
- }
- // Returns Compare(a + b, c);
- static int PlusCompare(const Bignum& a, const Bignum& b, const Bignum& c);
- // Returns a + b == c
- static bool PlusEqual(const Bignum& a, const Bignum& b, const Bignum& c) {
- return PlusCompare(a, b, c) == 0;
- }
- // Returns a + b <= c
- static bool PlusLessEqual(const Bignum& a, const Bignum& b, const Bignum& c) {
- return PlusCompare(a, b, c) <= 0;
- }
- // Returns a + b < c
- static bool PlusLess(const Bignum& a, const Bignum& b, const Bignum& c) {
- return PlusCompare(a, b, c) < 0;
- }
- private:
- typedef uint32_t Chunk;
- typedef uint64_t DoubleChunk;
-
- static const int kChunkSize = sizeof(Chunk) * 8;
- static const int kDoubleChunkSize = sizeof(DoubleChunk) * 8;
- // With bigit size of 28 we loose some bits, but a double still fits easily
- // into two chunks, and more importantly we can use the Comba multiplication.
- static const int kBigitSize = 28;
- static const Chunk kBigitMask = (1 << kBigitSize) - 1;
- // Every instance allocates kBigitLength chunks on the stack. Bignums cannot
- // grow. There are no checks if the stack-allocated space is sufficient.
- static const int kBigitCapacity = kMaxSignificantBits / kBigitSize;
-
- void EnsureCapacity(int size) {
- if (size > kBigitCapacity) {
- UNREACHABLE();
- }
- }
- void Align(const Bignum& other);
- void Clamp();
- bool IsClamped() const;
- void Zero();
- // Requires this to have enough capacity (no tests done).
- // Updates used_digits_ if necessary.
- // shift_amount must be < kBigitSize.
- void BigitsShiftLeft(int shift_amount);
- // BigitLength includes the "hidden" digits encoded in the exponent.
- int BigitLength() const { return used_digits_ + exponent_; }
- Chunk BigitAt(int index) const;
- void SubtractTimes(const Bignum& other, int factor);
-
- Chunk bigits_buffer_[kBigitCapacity];
- // A vector backed by bigits_buffer_. This way accesses to the array are
- // checked for out-of-bounds errors.
- Vector<Chunk> bigits_;
- int used_digits_;
- // The Bignum's value equals value(bigits_) * 2^(exponent_ * kBigitSize).
- int exponent_;
-
- DISALLOW_COPY_AND_ASSIGN(Bignum);
-};
-
-} // namespace double_conversion
-
-#endif // DOUBLE_CONVERSION_BIGNUM_H_
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/double-conversion/cached-powers.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/double-conversion/cached-powers.cc b/ext/kenlm/util/double-conversion/cached-powers.cc
deleted file mode 100644
index 9dcfa36..0000000
--- a/ext/kenlm/util/double-conversion/cached-powers.cc
+++ /dev/null
@@ -1,175 +0,0 @@
-// Copyright 2006-2008 the V8 project authors. All rights reserved.
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following
-// disclaimer in the documentation and/or other materials provided
-// with the distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived
-// from this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#include <cstdarg>
-#include <climits>
-#include <cmath>
-
-#include "utils.h"
-
-#include "cached-powers.h"
-
-namespace double_conversion {
-
-struct CachedPower {
- uint64_t significand;
- int16_t binary_exponent;
- int16_t decimal_exponent;
-};
-
-static const CachedPower kCachedPowers[] = {
- {UINT64_2PART_C(0xfa8fd5a0, 081c0288), -1220, -348},
- {UINT64_2PART_C(0xbaaee17f, a23ebf76), -1193, -340},
- {UINT64_2PART_C(0x8b16fb20, 3055ac76), -1166, -332},
- {UINT64_2PART_C(0xcf42894a, 5dce35ea), -1140, -324},
- {UINT64_2PART_C(0x9a6bb0aa, 55653b2d), -1113, -316},
- {UINT64_2PART_C(0xe61acf03, 3d1a45df), -1087, -308},
- {UINT64_2PART_C(0xab70fe17, c79ac6ca), -1060, -300},
- {UINT64_2PART_C(0xff77b1fc, bebcdc4f), -1034, -292},
- {UINT64_2PART_C(0xbe5691ef, 416bd60c), -1007, -284},
- {UINT64_2PART_C(0x8dd01fad, 907ffc3c), -980, -276},
- {UINT64_2PART_C(0xd3515c28, 31559a83), -954, -268},
- {UINT64_2PART_C(0x9d71ac8f, ada6c9b5), -927, -260},
- {UINT64_2PART_C(0xea9c2277, 23ee8bcb), -901, -252},
- {UINT64_2PART_C(0xaecc4991, 4078536d), -874, -244},
- {UINT64_2PART_C(0x823c1279, 5db6ce57), -847, -236},
- {UINT64_2PART_C(0xc2109436, 4dfb5637), -821, -228},
- {UINT64_2PART_C(0x9096ea6f, 3848984f), -794, -220},
- {UINT64_2PART_C(0xd77485cb, 25823ac7), -768, -212},
- {UINT64_2PART_C(0xa086cfcd, 97bf97f4), -741, -204},
- {UINT64_2PART_C(0xef340a98, 172aace5), -715, -196},
- {UINT64_2PART_C(0xb23867fb, 2a35b28e), -688, -188},
- {UINT64_2PART_C(0x84c8d4df, d2c63f3b), -661, -180},
- {UINT64_2PART_C(0xc5dd4427, 1ad3cdba), -635, -172},
- {UINT64_2PART_C(0x936b9fce, bb25c996), -608, -164},
- {UINT64_2PART_C(0xdbac6c24, 7d62a584), -582, -156},
- {UINT64_2PART_C(0xa3ab6658, 0d5fdaf6), -555, -148},
- {UINT64_2PART_C(0xf3e2f893, dec3f126), -529, -140},
- {UINT64_2PART_C(0xb5b5ada8, aaff80b8), -502, -132},
- {UINT64_2PART_C(0x87625f05, 6c7c4a8b), -475, -124},
- {UINT64_2PART_C(0xc9bcff60, 34c13053), -449, -116},
- {UINT64_2PART_C(0x964e858c, 91ba2655), -422, -108},
- {UINT64_2PART_C(0xdff97724, 70297ebd), -396, -100},
- {UINT64_2PART_C(0xa6dfbd9f, b8e5b88f), -369, -92},
- {UINT64_2PART_C(0xf8a95fcf, 88747d94), -343, -84},
- {UINT64_2PART_C(0xb9447093, 8fa89bcf), -316, -76},
- {UINT64_2PART_C(0x8a08f0f8, bf0f156b), -289, -68},
- {UINT64_2PART_C(0xcdb02555, 653131b6), -263, -60},
- {UINT64_2PART_C(0x993fe2c6, d07b7fac), -236, -52},
- {UINT64_2PART_C(0xe45c10c4, 2a2b3b06), -210, -44},
- {UINT64_2PART_C(0xaa242499, 697392d3), -183, -36},
- {UINT64_2PART_C(0xfd87b5f2, 8300ca0e), -157, -28},
- {UINT64_2PART_C(0xbce50864, 92111aeb), -130, -20},
- {UINT64_2PART_C(0x8cbccc09, 6f5088cc), -103, -12},
- {UINT64_2PART_C(0xd1b71758, e219652c), -77, -4},
- {UINT64_2PART_C(0x9c400000, 00000000), -50, 4},
- {UINT64_2PART_C(0xe8d4a510, 00000000), -24, 12},
- {UINT64_2PART_C(0xad78ebc5, ac620000), 3, 20},
- {UINT64_2PART_C(0x813f3978, f8940984), 30, 28},
- {UINT64_2PART_C(0xc097ce7b, c90715b3), 56, 36},
- {UINT64_2PART_C(0x8f7e32ce, 7bea5c70), 83, 44},
- {UINT64_2PART_C(0xd5d238a4, abe98068), 109, 52},
- {UINT64_2PART_C(0x9f4f2726, 179a2245), 136, 60},
- {UINT64_2PART_C(0xed63a231, d4c4fb27), 162, 68},
- {UINT64_2PART_C(0xb0de6538, 8cc8ada8), 189, 76},
- {UINT64_2PART_C(0x83c7088e, 1aab65db), 216, 84},
- {UINT64_2PART_C(0xc45d1df9, 42711d9a), 242, 92},
- {UINT64_2PART_C(0x924d692c, a61be758), 269, 100},
- {UINT64_2PART_C(0xda01ee64, 1a708dea), 295, 108},
- {UINT64_2PART_C(0xa26da399, 9aef774a), 322, 116},
- {UINT64_2PART_C(0xf209787b, b47d6b85), 348, 124},
- {UINT64_2PART_C(0xb454e4a1, 79dd1877), 375, 132},
- {UINT64_2PART_C(0x865b8692, 5b9bc5c2), 402, 140},
- {UINT64_2PART_C(0xc83553c5, c8965d3d), 428, 148},
- {UINT64_2PART_C(0x952ab45c, fa97a0b3), 455, 156},
- {UINT64_2PART_C(0xde469fbd, 99a05fe3), 481, 164},
- {UINT64_2PART_C(0xa59bc234, db398c25), 508, 172},
- {UINT64_2PART_C(0xf6c69a72, a3989f5c), 534, 180},
- {UINT64_2PART_C(0xb7dcbf53, 54e9bece), 561, 188},
- {UINT64_2PART_C(0x88fcf317, f22241e2), 588, 196},
- {UINT64_2PART_C(0xcc20ce9b, d35c78a5), 614, 204},
- {UINT64_2PART_C(0x98165af3, 7b2153df), 641, 212},
- {UINT64_2PART_C(0xe2a0b5dc, 971f303a), 667, 220},
- {UINT64_2PART_C(0xa8d9d153, 5ce3b396), 694, 228},
- {UINT64_2PART_C(0xfb9b7cd9, a4a7443c), 720, 236},
- {UINT64_2PART_C(0xbb764c4c, a7a44410), 747, 244},
- {UINT64_2PART_C(0x8bab8eef, b6409c1a), 774, 252},
- {UINT64_2PART_C(0xd01fef10, a657842c), 800, 260},
- {UINT64_2PART_C(0x9b10a4e5, e9913129), 827, 268},
- {UINT64_2PART_C(0xe7109bfb, a19c0c9d), 853, 276},
- {UINT64_2PART_C(0xac2820d9, 623bf429), 880, 284},
- {UINT64_2PART_C(0x80444b5e, 7aa7cf85), 907, 292},
- {UINT64_2PART_C(0xbf21e440, 03acdd2d), 933, 300},
- {UINT64_2PART_C(0x8e679c2f, 5e44ff8f), 960, 308},
- {UINT64_2PART_C(0xd433179d, 9c8cb841), 986, 316},
- {UINT64_2PART_C(0x9e19db92, b4e31ba9), 1013, 324},
- {UINT64_2PART_C(0xeb96bf6e, badf77d9), 1039, 332},
- {UINT64_2PART_C(0xaf87023b, 9bf0ee6b), 1066, 340},
-};
-
-static const int kCachedPowersLength = ARRAY_SIZE(kCachedPowers);
-static const int kCachedPowersOffset = 348; // -1 * the first decimal_exponent.
-static const double kD_1_LOG2_10 = 0.30102999566398114; // 1 / lg(10)
-// Difference between the decimal exponents in the table above.
-const int PowersOfTenCache::kDecimalExponentDistance = 8;
-const int PowersOfTenCache::kMinDecimalExponent = -348;
-const int PowersOfTenCache::kMaxDecimalExponent = 340;
-
-void PowersOfTenCache::GetCachedPowerForBinaryExponentRange(
- int min_exponent,
- int max_exponent,
- DiyFp* power,
- int* decimal_exponent) {
- int kQ = DiyFp::kSignificandSize;
- double k = ceil((min_exponent + kQ - 1) * kD_1_LOG2_10);
- int foo = kCachedPowersOffset;
- int index =
- (foo + static_cast<int>(k) - 1) / kDecimalExponentDistance + 1;
- ASSERT(0 <= index && index < kCachedPowersLength);
- CachedPower cached_power = kCachedPowers[index];
- ASSERT(min_exponent <= cached_power.binary_exponent);
- ASSERT(cached_power.binary_exponent <= max_exponent);
- *decimal_exponent = cached_power.decimal_exponent;
- *power = DiyFp(cached_power.significand, cached_power.binary_exponent);
-}
-
-
-void PowersOfTenCache::GetCachedPowerForDecimalExponent(int requested_exponent,
- DiyFp* power,
- int* found_exponent) {
- ASSERT(kMinDecimalExponent <= requested_exponent);
- ASSERT(requested_exponent < kMaxDecimalExponent + kDecimalExponentDistance);
- int index =
- (requested_exponent + kCachedPowersOffset) / kDecimalExponentDistance;
- CachedPower cached_power = kCachedPowers[index];
- *power = DiyFp(cached_power.significand, cached_power.binary_exponent);
- *found_exponent = cached_power.decimal_exponent;
- ASSERT(*found_exponent <= requested_exponent);
- ASSERT(requested_exponent < *found_exponent + kDecimalExponentDistance);
-}
-
-} // namespace double_conversion
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/double-conversion/cached-powers.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/double-conversion/cached-powers.h b/ext/kenlm/util/double-conversion/cached-powers.h
deleted file mode 100644
index 61a5061..0000000
--- a/ext/kenlm/util/double-conversion/cached-powers.h
+++ /dev/null
@@ -1,64 +0,0 @@
-// Copyright 2010 the V8 project authors. All rights reserved.
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following
-// disclaimer in the documentation and/or other materials provided
-// with the distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived
-// from this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#ifndef DOUBLE_CONVERSION_CACHED_POWERS_H_
-#define DOUBLE_CONVERSION_CACHED_POWERS_H_
-
-#include "diy-fp.h"
-
-namespace double_conversion {
-
-class PowersOfTenCache {
- public:
-
- // Not all powers of ten are cached. The decimal exponent of two neighboring
- // cached numbers will differ by kDecimalExponentDistance.
- static const int kDecimalExponentDistance;
-
- static const int kMinDecimalExponent;
- static const int kMaxDecimalExponent;
-
- // Returns a cached power-of-ten with a binary exponent in the range
- // [min_exponent; max_exponent] (boundaries included).
- static void GetCachedPowerForBinaryExponentRange(int min_exponent,
- int max_exponent,
- DiyFp* power,
- int* decimal_exponent);
-
- // Returns a cached power of ten x ~= 10^k such that
- // k <= decimal_exponent < k + kCachedPowersDecimalDistance.
- // The given decimal_exponent must satisfy
- // kMinDecimalExponent <= requested_exponent, and
- // requested_exponent < kMaxDecimalExponent + kDecimalExponentDistance.
- static void GetCachedPowerForDecimalExponent(int requested_exponent,
- DiyFp* power,
- int* found_exponent);
-};
-
-} // namespace double_conversion
-
-#endif // DOUBLE_CONVERSION_CACHED_POWERS_H_
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/double-conversion/diy-fp.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/double-conversion/diy-fp.cc b/ext/kenlm/util/double-conversion/diy-fp.cc
deleted file mode 100644
index ddd1891..0000000
--- a/ext/kenlm/util/double-conversion/diy-fp.cc
+++ /dev/null
@@ -1,57 +0,0 @@
-// Copyright 2010 the V8 project authors. All rights reserved.
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following
-// disclaimer in the documentation and/or other materials provided
-// with the distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived
-// from this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
-#include "diy-fp.h"
-#include "utils.h"
-
-namespace double_conversion {
-
-void DiyFp::Multiply(const DiyFp& other) {
- // Simply "emulates" a 128 bit multiplication.
- // However: the resulting number only contains 64 bits. The least
- // significant 64 bits are only used for rounding the most significant 64
- // bits.
- const uint64_t kM32 = 0xFFFFFFFFU;
- uint64_t a = f_ >> 32;
- uint64_t b = f_ & kM32;
- uint64_t c = other.f_ >> 32;
- uint64_t d = other.f_ & kM32;
- uint64_t ac = a * c;
- uint64_t bc = b * c;
- uint64_t ad = a * d;
- uint64_t bd = b * d;
- uint64_t tmp = (bd >> 32) + (ad & kM32) + (bc & kM32);
- // By adding 1U << 31 to tmp we round the final result.
- // Halfway cases will be round up.
- tmp += 1U << 31;
- uint64_t result_f = ac + (ad >> 32) + (bc >> 32) + (tmp >> 32);
- e_ += other.e_ + 64;
- f_ = result_f;
-}
-
-} // namespace double_conversion
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/double-conversion/diy-fp.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/double-conversion/diy-fp.h b/ext/kenlm/util/double-conversion/diy-fp.h
deleted file mode 100644
index 9dcf8fb..0000000
--- a/ext/kenlm/util/double-conversion/diy-fp.h
+++ /dev/null
@@ -1,118 +0,0 @@
-// Copyright 2010 the V8 project authors. All rights reserved.
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following
-// disclaimer in the documentation and/or other materials provided
-// with the distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived
-// from this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#ifndef DOUBLE_CONVERSION_DIY_FP_H_
-#define DOUBLE_CONVERSION_DIY_FP_H_
-
-#include "utils.h"
-
-namespace double_conversion {
-
-// This "Do It Yourself Floating Point" class implements a floating-point number
-// with a uint64 significand and an int exponent. Normalized DiyFp numbers will
-// have the most significant bit of the significand set.
-// Multiplication and Subtraction do not normalize their results.
-// DiyFp are not designed to contain special doubles (NaN and Infinity).
-class DiyFp {
- public:
- static const int kSignificandSize = 64;
-
- DiyFp() : f_(0), e_(0) {}
- DiyFp(uint64_t f, int e) : f_(f), e_(e) {}
-
- // this = this - other.
- // The exponents of both numbers must be the same and the significand of this
- // must be bigger than the significand of other.
- // The result will not be normalized.
- void Subtract(const DiyFp& other) {
- ASSERT(e_ == other.e_);
- ASSERT(f_ >= other.f_);
- f_ -= other.f_;
- }
-
- // Returns a - b.
- // The exponents of both numbers must be the same and this must be bigger
- // than other. The result will not be normalized.
- static DiyFp Minus(const DiyFp& a, const DiyFp& b) {
- DiyFp result = a;
- result.Subtract(b);
- return result;
- }
-
-
- // this = this * other.
- void Multiply(const DiyFp& other);
-
- // returns a * b;
- static DiyFp Times(const DiyFp& a, const DiyFp& b) {
- DiyFp result = a;
- result.Multiply(b);
- return result;
- }
-
- void Normalize() {
- ASSERT(f_ != 0);
- uint64_t f = f_;
- int e = e_;
-
- // This method is mainly called for normalizing boundaries. In general
- // boundaries need to be shifted by 10 bits. We thus optimize for this case.
- const uint64_t k10MSBits = UINT64_2PART_C(0xFFC00000, 00000000);
- while ((f & k10MSBits) == 0) {
- f <<= 10;
- e -= 10;
- }
- while ((f & kUint64MSB) == 0) {
- f <<= 1;
- e--;
- }
- f_ = f;
- e_ = e;
- }
-
- static DiyFp Normalize(const DiyFp& a) {
- DiyFp result = a;
- result.Normalize();
- return result;
- }
-
- uint64_t f() const { return f_; }
- int e() const { return e_; }
-
- void set_f(uint64_t new_value) { f_ = new_value; }
- void set_e(int new_value) { e_ = new_value; }
-
- private:
- static const uint64_t kUint64MSB = UINT64_2PART_C(0x80000000, 00000000);
-
- uint64_t f_;
- int e_;
-};
-
-} // namespace double_conversion
-
-#endif // DOUBLE_CONVERSION_DIY_FP_H_
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/double-conversion/double-conversion.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/double-conversion/double-conversion.cc b/ext/kenlm/util/double-conversion/double-conversion.cc
deleted file mode 100644
index 8a7923c..0000000
--- a/ext/kenlm/util/double-conversion/double-conversion.cc
+++ /dev/null
@@ -1,889 +0,0 @@
-// Copyright 2010 the V8 project authors. All rights reserved.
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following
-// disclaimer in the documentation and/or other materials provided
-// with the distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived
-// from this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#include <climits>
-#include <cmath>
-
-#include "double-conversion.h"
-
-#include "bignum-dtoa.h"
-#include "fast-dtoa.h"
-#include "fixed-dtoa.h"
-#include "ieee.h"
-#include "strtod.h"
-#include "utils.h"
-
-namespace double_conversion {
-
-const DoubleToStringConverter& DoubleToStringConverter::EcmaScriptConverter() {
- int flags = UNIQUE_ZERO | EMIT_POSITIVE_EXPONENT_SIGN;
- static DoubleToStringConverter converter(flags,
- "Infinity",
- "NaN",
- 'e',
- -6, 21,
- 6, 0);
- return converter;
-}
-
-
-bool DoubleToStringConverter::HandleSpecialValues(
- double value,
- StringBuilder* result_builder) const {
- Double double_inspect(value);
- if (double_inspect.IsInfinite()) {
- if (infinity_symbol_ == NULL) return false;
- if (value < 0) {
- result_builder->AddCharacter('-');
- }
- result_builder->AddString(infinity_symbol_);
- return true;
- }
- if (double_inspect.IsNan()) {
- if (nan_symbol_ == NULL) return false;
- result_builder->AddString(nan_symbol_);
- return true;
- }
- return false;
-}
-
-
-void DoubleToStringConverter::CreateExponentialRepresentation(
- const char* decimal_digits,
- int length,
- int exponent,
- StringBuilder* result_builder) const {
- ASSERT(length != 0);
- result_builder->AddCharacter(decimal_digits[0]);
- if (length != 1) {
- result_builder->AddCharacter('.');
- result_builder->AddSubstring(&decimal_digits[1], length-1);
- }
- result_builder->AddCharacter(exponent_character_);
- if (exponent < 0) {
- result_builder->AddCharacter('-');
- exponent = -exponent;
- } else {
- if ((flags_ & EMIT_POSITIVE_EXPONENT_SIGN) != 0) {
- result_builder->AddCharacter('+');
- }
- }
- if (exponent == 0) {
- result_builder->AddCharacter('0');
- return;
- }
- ASSERT(exponent < 1e4);
- const int kMaxExponentLength = 5;
- char buffer[kMaxExponentLength + 1];
- buffer[kMaxExponentLength] = '\0';
- int first_char_pos = kMaxExponentLength;
- while (exponent > 0) {
- buffer[--first_char_pos] = '0' + (exponent % 10);
- exponent /= 10;
- }
- result_builder->AddSubstring(&buffer[first_char_pos],
- kMaxExponentLength - first_char_pos);
-}
-
-
-void DoubleToStringConverter::CreateDecimalRepresentation(
- const char* decimal_digits,
- int length,
- int decimal_point,
- int digits_after_point,
- StringBuilder* result_builder) const {
- // Create a representation that is padded with zeros if needed.
- if (decimal_point <= 0) {
- // "0.00000decimal_rep".
- result_builder->AddCharacter('0');
- if (digits_after_point > 0) {
- result_builder->AddCharacter('.');
- result_builder->AddPadding('0', -decimal_point);
- ASSERT(length <= digits_after_point - (-decimal_point));
- result_builder->AddSubstring(decimal_digits, length);
- int remaining_digits = digits_after_point - (-decimal_point) - length;
- result_builder->AddPadding('0', remaining_digits);
- }
- } else if (decimal_point >= length) {
- // "decimal_rep0000.00000" or "decimal_rep.0000"
- result_builder->AddSubstring(decimal_digits, length);
- result_builder->AddPadding('0', decimal_point - length);
- if (digits_after_point > 0) {
- result_builder->AddCharacter('.');
- result_builder->AddPadding('0', digits_after_point);
- }
- } else {
- // "decima.l_rep000"
- ASSERT(digits_after_point > 0);
- result_builder->AddSubstring(decimal_digits, decimal_point);
- result_builder->AddCharacter('.');
- ASSERT(length - decimal_point <= digits_after_point);
- result_builder->AddSubstring(&decimal_digits[decimal_point],
- length - decimal_point);
- int remaining_digits = digits_after_point - (length - decimal_point);
- result_builder->AddPadding('0', remaining_digits);
- }
- if (digits_after_point == 0) {
- if ((flags_ & EMIT_TRAILING_DECIMAL_POINT) != 0) {
- result_builder->AddCharacter('.');
- }
- if ((flags_ & EMIT_TRAILING_ZERO_AFTER_POINT) != 0) {
- result_builder->AddCharacter('0');
- }
- }
-}
-
-
-bool DoubleToStringConverter::ToShortestIeeeNumber(
- double value,
- StringBuilder* result_builder,
- DoubleToStringConverter::DtoaMode mode) const {
- ASSERT(mode == SHORTEST || mode == SHORTEST_SINGLE);
- if (Double(value).IsSpecial()) {
- return HandleSpecialValues(value, result_builder);
- }
-
- int decimal_point;
- bool sign;
- const int kDecimalRepCapacity = kBase10MaximalLength + 1;
- char decimal_rep[kDecimalRepCapacity];
- int decimal_rep_length;
-
- DoubleToAscii(value, mode, 0, decimal_rep, kDecimalRepCapacity,
- &sign, &decimal_rep_length, &decimal_point);
-
- bool unique_zero = (flags_ & UNIQUE_ZERO) != 0;
- if (sign && (value != 0.0 || !unique_zero)) {
- result_builder->AddCharacter('-');
- }
-
- int exponent = decimal_point - 1;
- if ((decimal_in_shortest_low_ <= exponent) &&
- (exponent < decimal_in_shortest_high_)) {
- CreateDecimalRepresentation(decimal_rep, decimal_rep_length,
- decimal_point,
- Max(0, decimal_rep_length - decimal_point),
- result_builder);
- } else {
- CreateExponentialRepresentation(decimal_rep, decimal_rep_length, exponent,
- result_builder);
- }
- return true;
-}
-
-
-bool DoubleToStringConverter::ToFixed(double value,
- int requested_digits,
- StringBuilder* result_builder) const {
- ASSERT(kMaxFixedDigitsBeforePoint == 60);
- const double kFirstNonFixed = 1e60;
-
- if (Double(value).IsSpecial()) {
- return HandleSpecialValues(value, result_builder);
- }
-
- if (requested_digits > kMaxFixedDigitsAfterPoint) return false;
- if (value >= kFirstNonFixed || value <= -kFirstNonFixed) return false;
-
- // Find a sufficiently precise decimal representation of n.
- int decimal_point;
- bool sign;
- // Add space for the '\0' byte.
- const int kDecimalRepCapacity =
- kMaxFixedDigitsBeforePoint + kMaxFixedDigitsAfterPoint + 1;
- char decimal_rep[kDecimalRepCapacity];
- int decimal_rep_length;
- DoubleToAscii(value, FIXED, requested_digits,
- decimal_rep, kDecimalRepCapacity,
- &sign, &decimal_rep_length, &decimal_point);
-
- bool unique_zero = ((flags_ & UNIQUE_ZERO) != 0);
- if (sign && (value != 0.0 || !unique_zero)) {
- result_builder->AddCharacter('-');
- }
-
- CreateDecimalRepresentation(decimal_rep, decimal_rep_length, decimal_point,
- requested_digits, result_builder);
- return true;
-}
-
-
-bool DoubleToStringConverter::ToExponential(
- double value,
- int requested_digits,
- StringBuilder* result_builder) const {
- if (Double(value).IsSpecial()) {
- return HandleSpecialValues(value, result_builder);
- }
-
- if (requested_digits < -1) return false;
- if (requested_digits > kMaxExponentialDigits) return false;
-
- int decimal_point;
- bool sign;
- // Add space for digit before the decimal point and the '\0' character.
- const int kDecimalRepCapacity = kMaxExponentialDigits + 2;
- ASSERT(kDecimalRepCapacity > kBase10MaximalLength);
- char decimal_rep[kDecimalRepCapacity];
- int decimal_rep_length;
-
- if (requested_digits == -1) {
- DoubleToAscii(value, SHORTEST, 0,
- decimal_rep, kDecimalRepCapacity,
- &sign, &decimal_rep_length, &decimal_point);
- } else {
- DoubleToAscii(value, PRECISION, requested_digits + 1,
- decimal_rep, kDecimalRepCapacity,
- &sign, &decimal_rep_length, &decimal_point);
- ASSERT(decimal_rep_length <= requested_digits + 1);
-
- for (int i = decimal_rep_length; i < requested_digits + 1; ++i) {
- decimal_rep[i] = '0';
- }
- decimal_rep_length = requested_digits + 1;
- }
-
- bool unique_zero = ((flags_ & UNIQUE_ZERO) != 0);
- if (sign && (value != 0.0 || !unique_zero)) {
- result_builder->AddCharacter('-');
- }
-
- int exponent = decimal_point - 1;
- CreateExponentialRepresentation(decimal_rep,
- decimal_rep_length,
- exponent,
- result_builder);
- return true;
-}
-
-
-bool DoubleToStringConverter::ToPrecision(double value,
- int precision,
- StringBuilder* result_builder) const {
- if (Double(value).IsSpecial()) {
- return HandleSpecialValues(value, result_builder);
- }
-
- if (precision < kMinPrecisionDigits || precision > kMaxPrecisionDigits) {
- return false;
- }
-
- // Find a sufficiently precise decimal representation of n.
- int decimal_point;
- bool sign;
- // Add one for the terminating null character.
- const int kDecimalRepCapacity = kMaxPrecisionDigits + 1;
- char decimal_rep[kDecimalRepCapacity];
- int decimal_rep_length;
-
- DoubleToAscii(value, PRECISION, precision,
- decimal_rep, kDecimalRepCapacity,
- &sign, &decimal_rep_length, &decimal_point);
- ASSERT(decimal_rep_length <= precision);
-
- bool unique_zero = ((flags_ & UNIQUE_ZERO) != 0);
- if (sign && (value != 0.0 || !unique_zero)) {
- result_builder->AddCharacter('-');
- }
-
- // The exponent if we print the number as x.xxeyyy. That is with the
- // decimal point after the first digit.
- int exponent = decimal_point - 1;
-
- int extra_zero = ((flags_ & EMIT_TRAILING_ZERO_AFTER_POINT) != 0) ? 1 : 0;
- if ((-decimal_point + 1 > max_leading_padding_zeroes_in_precision_mode_) ||
- (decimal_point - precision + extra_zero >
- max_trailing_padding_zeroes_in_precision_mode_)) {
- // Fill buffer to contain 'precision' digits.
- // Usually the buffer is already at the correct length, but 'DoubleToAscii'
- // is allowed to return less characters.
- for (int i = decimal_rep_length; i < precision; ++i) {
- decimal_rep[i] = '0';
- }
-
- CreateExponentialRepresentation(decimal_rep,
- precision,
- exponent,
- result_builder);
- } else {
- CreateDecimalRepresentation(decimal_rep, decimal_rep_length, decimal_point,
- Max(0, precision - decimal_point),
- result_builder);
- }
- return true;
-}
-
-
-static BignumDtoaMode DtoaToBignumDtoaMode(
- DoubleToStringConverter::DtoaMode dtoa_mode) {
- switch (dtoa_mode) {
- case DoubleToStringConverter::SHORTEST: return BIGNUM_DTOA_SHORTEST;
- case DoubleToStringConverter::SHORTEST_SINGLE:
- return BIGNUM_DTOA_SHORTEST_SINGLE;
- case DoubleToStringConverter::FIXED: return BIGNUM_DTOA_FIXED;
- case DoubleToStringConverter::PRECISION: return BIGNUM_DTOA_PRECISION;
- default:
- UNREACHABLE();
- return BIGNUM_DTOA_SHORTEST; // To silence compiler.
- }
-}
-
-
-void DoubleToStringConverter::DoubleToAscii(double v,
- DtoaMode mode,
- int requested_digits,
- char* buffer,
- int buffer_length,
- bool* sign,
- int* length,
- int* point) {
- Vector<char> vector(buffer, buffer_length);
- ASSERT(!Double(v).IsSpecial());
- ASSERT(mode == SHORTEST || mode == SHORTEST_SINGLE || requested_digits >= 0);
-
- if (Double(v).Sign() < 0) {
- *sign = true;
- v = -v;
- } else {
- *sign = false;
- }
-
- if (mode == PRECISION && requested_digits == 0) {
- vector[0] = '\0';
- *length = 0;
- return;
- }
-
- if (v == 0) {
- vector[0] = '0';
- vector[1] = '\0';
- *length = 1;
- *point = 1;
- return;
- }
-
- bool fast_worked;
- switch (mode) {
- case SHORTEST:
- fast_worked = FastDtoa(v, FAST_DTOA_SHORTEST, 0, vector, length, point);
- break;
- case SHORTEST_SINGLE:
- fast_worked = FastDtoa(v, FAST_DTOA_SHORTEST_SINGLE, 0,
- vector, length, point);
- break;
- case FIXED:
- fast_worked = FastFixedDtoa(v, requested_digits, vector, length, point);
- break;
- case PRECISION:
- fast_worked = FastDtoa(v, FAST_DTOA_PRECISION, requested_digits,
- vector, length, point);
- break;
- default:
- UNREACHABLE();
- fast_worked = false;
- }
- if (fast_worked) return;
-
- // If the fast dtoa didn't succeed use the slower bignum version.
- BignumDtoaMode bignum_mode = DtoaToBignumDtoaMode(mode);
- BignumDtoa(v, bignum_mode, requested_digits, vector, length, point);
- vector[*length] = '\0';
-}
-
-
-// Consumes the given substring from the iterator.
-// Returns false, if the substring does not match.
-static bool ConsumeSubString(const char** current,
- const char* end,
- const char* substring) {
- ASSERT(**current == *substring);
- for (substring++; *substring != '\0'; substring++) {
- ++*current;
- if (*current == end || **current != *substring) return false;
- }
- ++*current;
- return true;
-}
-
-
-// Maximum number of significant digits in decimal representation.
-// The longest possible double in decimal representation is
-// (2^53 - 1) * 2 ^ -1074 that is (2 ^ 53 - 1) * 5 ^ 1074 / 10 ^ 1074
-// (768 digits). If we parse a number whose first digits are equal to a
-// mean of 2 adjacent doubles (that could have up to 769 digits) the result
-// must be rounded to the bigger one unless the tail consists of zeros, so
-// we don't need to preserve all the digits.
-const int kMaxSignificantDigits = 772;
-
-
-// Returns true if a nonspace found and false if the end has reached.
-static inline bool AdvanceToNonspace(const char** current, const char* end) {
- while (*current != end) {
- if (**current != ' ') return true;
- ++*current;
- }
- return false;
-}
-
-
-static bool isDigit(int x, int radix) {
- return (x >= '0' && x <= '9' && x < '0' + radix)
- || (radix > 10 && x >= 'a' && x < 'a' + radix - 10)
- || (radix > 10 && x >= 'A' && x < 'A' + radix - 10);
-}
-
-
-static double SignedZero(bool sign) {
- return sign ? -0.0 : 0.0;
-}
-
-
-// Parsing integers with radix 2, 4, 8, 16, 32. Assumes current != end.
-template <int radix_log_2>
-static double RadixStringToIeee(const char* current,
- const char* end,
- bool sign,
- bool allow_trailing_junk,
- double junk_string_value,
- bool read_as_double,
- const char** trailing_pointer) {
- ASSERT(current != end);
-
- const int kDoubleSize = Double::kSignificandSize;
- const int kSingleSize = Single::kSignificandSize;
- const int kSignificandSize = read_as_double? kDoubleSize: kSingleSize;
-
- // Skip leading 0s.
- while (*current == '0') {
- ++current;
- if (current == end) {
- *trailing_pointer = end;
- return SignedZero(sign);
- }
- }
-
- int64_t number = 0;
- int exponent = 0;
- const int radix = (1 << radix_log_2);
-
- do {
- int digit;
- if (*current >= '0' && *current <= '9' && *current < '0' + radix) {
- digit = static_cast<char>(*current) - '0';
- } else if (radix > 10 && *current >= 'a' && *current < 'a' + radix - 10) {
- digit = static_cast<char>(*current) - 'a' + 10;
- } else if (radix > 10 && *current >= 'A' && *current < 'A' + radix - 10) {
- digit = static_cast<char>(*current) - 'A' + 10;
- } else {
- if (allow_trailing_junk || !AdvanceToNonspace(¤t, end)) {
- break;
- } else {
- return junk_string_value;
- }
- }
-
- number = number * radix + digit;
- int overflow = static_cast<int>(number >> kSignificandSize);
- if (overflow != 0) {
- // Overflow occurred. Need to determine which direction to round the
- // result.
- int overflow_bits_count = 1;
- while (overflow > 1) {
- overflow_bits_count++;
- overflow >>= 1;
- }
-
- int dropped_bits_mask = ((1 << overflow_bits_count) - 1);
- int dropped_bits = static_cast<int>(number) & dropped_bits_mask;
- number >>= overflow_bits_count;
- exponent = overflow_bits_count;
-
- bool zero_tail = true;
- while (true) {
- ++current;
- if (current == end || !isDigit(*current, radix)) break;
- zero_tail = zero_tail && *current == '0';
- exponent += radix_log_2;
- }
-
- if (!allow_trailing_junk && AdvanceToNonspace(¤t, end)) {
- return junk_string_value;
- }
-
- int middle_value = (1 << (overflow_bits_count - 1));
- if (dropped_bits > middle_value) {
- number++; // Rounding up.
- } else if (dropped_bits == middle_value) {
- // Rounding to even to consistency with decimals: half-way case rounds
- // up if significant part is odd and down otherwise.
- if ((number & 1) != 0 || !zero_tail) {
- number++; // Rounding up.
- }
- }
-
- // Rounding up may cause overflow.
- if ((number & ((int64_t)1 << kSignificandSize)) != 0) {
- exponent++;
- number >>= 1;
- }
- break;
- }
- ++current;
- } while (current != end);
-
- ASSERT(number < ((int64_t)1 << kSignificandSize));
- ASSERT(static_cast<int64_t>(static_cast<double>(number)) == number);
-
- *trailing_pointer = current;
-
- if (exponent == 0) {
- if (sign) {
- if (number == 0) return -0.0;
- number = -number;
- }
- return static_cast<double>(number);
- }
-
- ASSERT(number != 0);
- return Double(DiyFp(number, exponent)).value();
-}
-
-
-double StringToDoubleConverter::StringToIeee(
- const char* input,
- int length,
- int* processed_characters_count,
- bool read_as_double) const {
- const char* current = input;
- const char* end = input + length;
-
- *processed_characters_count = 0;
-
- const bool allow_trailing_junk = (flags_ & ALLOW_TRAILING_JUNK) != 0;
- const bool allow_leading_spaces = (flags_ & ALLOW_LEADING_SPACES) != 0;
- const bool allow_trailing_spaces = (flags_ & ALLOW_TRAILING_SPACES) != 0;
- const bool allow_spaces_after_sign = (flags_ & ALLOW_SPACES_AFTER_SIGN) != 0;
-
- // To make sure that iterator dereferencing is valid the following
- // convention is used:
- // 1. Each '++current' statement is followed by check for equality to 'end'.
- // 2. If AdvanceToNonspace returned false then current == end.
- // 3. If 'current' becomes equal to 'end' the function returns or goes to
- // 'parsing_done'.
- // 4. 'current' is not dereferenced after the 'parsing_done' label.
- // 5. Code before 'parsing_done' may rely on 'current != end'.
- if (current == end) return empty_string_value_;
-
- if (allow_leading_spaces || allow_trailing_spaces) {
- if (!AdvanceToNonspace(¤t, end)) {
- *processed_characters_count = current - input;
- return empty_string_value_;
- }
- if (!allow_leading_spaces && (input != current)) {
- // No leading spaces allowed, but AdvanceToNonspace moved forward.
- return junk_string_value_;
- }
- }
-
- // The longest form of simplified number is: "-<significant digits>.1eXXX\0".
- const int kBufferSize = kMaxSignificantDigits + 10;
- char buffer[kBufferSize]; // NOLINT: size is known at compile time.
- int buffer_pos = 0;
-
- // Exponent will be adjusted if insignificant digits of the integer part
- // or insignificant leading zeros of the fractional part are dropped.
- int exponent = 0;
- int significant_digits = 0;
- int insignificant_digits = 0;
- bool nonzero_digit_dropped = false;
-
- bool sign = false;
-
- if (*current == '+' || *current == '-') {
- sign = (*current == '-');
- ++current;
- const char* next_non_space = current;
- // Skip following spaces (if allowed).
- if (!AdvanceToNonspace(&next_non_space, end)) return junk_string_value_;
- if (!allow_spaces_after_sign && (current != next_non_space)) {
- return junk_string_value_;
- }
- current = next_non_space;
- }
-
- if (infinity_symbol_ != NULL) {
- if (*current == infinity_symbol_[0]) {
- if (!ConsumeSubString(¤t, end, infinity_symbol_)) {
- return junk_string_value_;
- }
-
- if (!(allow_trailing_spaces || allow_trailing_junk) && (current != end)) {
- return junk_string_value_;
- }
- if (!allow_trailing_junk && AdvanceToNonspace(¤t, end)) {
- return junk_string_value_;
- }
-
- ASSERT(buffer_pos == 0);
- *processed_characters_count = current - input;
- return sign ? -Double::Infinity() : Double::Infinity();
- }
- }
-
- if (nan_symbol_ != NULL) {
- if (*current == nan_symbol_[0]) {
- if (!ConsumeSubString(¤t, end, nan_symbol_)) {
- return junk_string_value_;
- }
-
- if (!(allow_trailing_spaces || allow_trailing_junk) && (current != end)) {
- return junk_string_value_;
- }
- if (!allow_trailing_junk && AdvanceToNonspace(¤t, end)) {
- return junk_string_value_;
- }
-
- ASSERT(buffer_pos == 0);
- *processed_characters_count = current - input;
- return sign ? -Double::NaN() : Double::NaN();
- }
- }
-
- bool leading_zero = false;
- if (*current == '0') {
- ++current;
- if (current == end) {
- *processed_characters_count = current - input;
- return SignedZero(sign);
- }
-
- leading_zero = true;
-
- // It could be hexadecimal value.
- if ((flags_ & ALLOW_HEX) && (*current == 'x' || *current == 'X')) {
- ++current;
- if (current == end || !isDigit(*current, 16)) {
- return junk_string_value_; // "0x".
- }
-
- const char* tail_pointer = NULL;
- double result = RadixStringToIeee<4>(current,
- end,
- sign,
- allow_trailing_junk,
- junk_string_value_,
- read_as_double,
- &tail_pointer);
- if (tail_pointer != NULL) {
- if (allow_trailing_spaces) AdvanceToNonspace(&tail_pointer, end);
- *processed_characters_count = tail_pointer - input;
- }
- return result;
- }
-
- // Ignore leading zeros in the integer part.
- while (*current == '0') {
- ++current;
- if (current == end) {
- *processed_characters_count = current - input;
- return SignedZero(sign);
- }
- }
- }
-
- bool octal = leading_zero && (flags_ & ALLOW_OCTALS) != 0;
-
- // Copy significant digits of the integer part (if any) to the buffer.
- while (*current >= '0' && *current <= '9') {
- if (significant_digits < kMaxSignificantDigits) {
- ASSERT(buffer_pos < kBufferSize);
- buffer[buffer_pos++] = static_cast<char>(*current);
- significant_digits++;
- // Will later check if it's an octal in the buffer.
- } else {
- insignificant_digits++; // Move the digit into the exponential part.
- nonzero_digit_dropped = nonzero_digit_dropped || *current != '0';
- }
- octal = octal && *current < '8';
- ++current;
- if (current == end) goto parsing_done;
- }
-
- if (significant_digits == 0) {
- octal = false;
- }
-
- if (*current == '.') {
- if (octal && !allow_trailing_junk) return junk_string_value_;
- if (octal) goto parsing_done;
-
- ++current;
- if (current == end) {
- if (significant_digits == 0 && !leading_zero) {
- return junk_string_value_;
- } else {
- goto parsing_done;
- }
- }
-
- if (significant_digits == 0) {
- // octal = false;
- // Integer part consists of 0 or is absent. Significant digits start after
- // leading zeros (if any).
- while (*current == '0') {
- ++current;
- if (current == end) {
- *processed_characters_count = current - input;
- return SignedZero(sign);
- }
- exponent--; // Move this 0 into the exponent.
- }
- }
-
- // There is a fractional part.
- // We don't emit a '.', but adjust the exponent instead.
- while (*current >= '0' && *current <= '9') {
- if (significant_digits < kMaxSignificantDigits) {
- ASSERT(buffer_pos < kBufferSize);
- buffer[buffer_pos++] = static_cast<char>(*current);
- significant_digits++;
- exponent--;
- } else {
- // Ignore insignificant digits in the fractional part.
- nonzero_digit_dropped = nonzero_digit_dropped || *current != '0';
- }
- ++current;
- if (current == end) goto parsing_done;
- }
- }
-
- if (!leading_zero && exponent == 0 && significant_digits == 0) {
- // If leading_zeros is true then the string contains zeros.
- // If exponent < 0 then string was [+-]\.0*...
- // If significant_digits != 0 the string is not equal to 0.
- // Otherwise there are no digits in the string.
- return junk_string_value_;
- }
-
- // Parse exponential part.
- if (*current == 'e' || *current == 'E') {
- if (octal && !allow_trailing_junk) return junk_string_value_;
- if (octal) goto parsing_done;
- ++current;
- if (current == end) {
- if (allow_trailing_junk) {
- goto parsing_done;
- } else {
- return junk_string_value_;
- }
- }
- char sign = '+';
- if (*current == '+' || *current == '-') {
- sign = static_cast<char>(*current);
- ++current;
- if (current == end) {
- if (allow_trailing_junk) {
- goto parsing_done;
- } else {
- return junk_string_value_;
- }
- }
- }
-
- if (current == end || *current < '0' || *current > '9') {
- if (allow_trailing_junk) {
- goto parsing_done;
- } else {
- return junk_string_value_;
- }
- }
-
- const int max_exponent = INT_MAX / 2;
- ASSERT(-max_exponent / 2 <= exponent && exponent <= max_exponent / 2);
- int num = 0;
- do {
- // Check overflow.
- int digit = *current - '0';
- if (num >= max_exponent / 10
- && !(num == max_exponent / 10 && digit <= max_exponent % 10)) {
- num = max_exponent;
- } else {
- num = num * 10 + digit;
- }
- ++current;
- } while (current != end && *current >= '0' && *current <= '9');
-
- exponent += (sign == '-' ? -num : num);
- }
-
- if (!(allow_trailing_spaces || allow_trailing_junk) && (current != end)) {
- return junk_string_value_;
- }
- if (!allow_trailing_junk && AdvanceToNonspace(¤t, end)) {
- return junk_string_value_;
- }
- if (allow_trailing_spaces) {
- AdvanceToNonspace(¤t, end);
- }
-
- parsing_done:
- exponent += insignificant_digits;
-
- if (octal) {
- double result;
- const char* tail_pointer = NULL;
- result = RadixStringToIeee<3>(buffer,
- buffer + buffer_pos,
- sign,
- allow_trailing_junk,
- junk_string_value_,
- read_as_double,
- &tail_pointer);
- ASSERT(tail_pointer != NULL);
- *processed_characters_count = current - input;
- return result;
- }
-
- if (nonzero_digit_dropped) {
- buffer[buffer_pos++] = '1';
- exponent--;
- }
-
- ASSERT(buffer_pos < kBufferSize);
- buffer[buffer_pos] = '\0';
-
- double converted;
- if (read_as_double) {
- converted = Strtod(Vector<const char>(buffer, buffer_pos), exponent);
- } else {
- converted = Strtof(Vector<const char>(buffer, buffer_pos), exponent);
- }
- *processed_characters_count = current - input;
- return sign? -converted: converted;
-}
-
-} // namespace double_conversion
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/double-conversion/double-conversion.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/double-conversion/double-conversion.h b/ext/kenlm/util/double-conversion/double-conversion.h
deleted file mode 100644
index 1c3387d..0000000
--- a/ext/kenlm/util/double-conversion/double-conversion.h
+++ /dev/null
@@ -1,536 +0,0 @@
-// Copyright 2012 the V8 project authors. All rights reserved.
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following
-// disclaimer in the documentation and/or other materials provided
-// with the distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived
-// from this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#ifndef DOUBLE_CONVERSION_DOUBLE_CONVERSION_H_
-#define DOUBLE_CONVERSION_DOUBLE_CONVERSION_H_
-
-#include "utils.h"
-
-namespace double_conversion {
-
-class DoubleToStringConverter {
- public:
- // When calling ToFixed with a double > 10^kMaxFixedDigitsBeforePoint
- // or a requested_digits parameter > kMaxFixedDigitsAfterPoint then the
- // function returns false.
- static const int kMaxFixedDigitsBeforePoint = 60;
- static const int kMaxFixedDigitsAfterPoint = 60;
-
- // When calling ToExponential with a requested_digits
- // parameter > kMaxExponentialDigits then the function returns false.
- static const int kMaxExponentialDigits = 120;
-
- // When calling ToPrecision with a requested_digits
- // parameter < kMinPrecisionDigits or requested_digits > kMaxPrecisionDigits
- // then the function returns false.
- static const int kMinPrecisionDigits = 1;
- static const int kMaxPrecisionDigits = 120;
-
- enum Flags {
- NO_FLAGS = 0,
- EMIT_POSITIVE_EXPONENT_SIGN = 1,
- EMIT_TRAILING_DECIMAL_POINT = 2,
- EMIT_TRAILING_ZERO_AFTER_POINT = 4,
- UNIQUE_ZERO = 8
- };
-
- // Flags should be a bit-or combination of the possible Flags-enum.
- // - NO_FLAGS: no special flags.
- // - EMIT_POSITIVE_EXPONENT_SIGN: when the number is converted into exponent
- // form, emits a '+' for positive exponents. Example: 1.2e+2.
- // - EMIT_TRAILING_DECIMAL_POINT: when the input number is an integer and is
- // converted into decimal format then a trailing decimal point is appended.
- // Example: 2345.0 is converted to "2345.".
- // - EMIT_TRAILING_ZERO_AFTER_POINT: in addition to a trailing decimal point
- // emits a trailing '0'-character. This flag requires the
- // EXMIT_TRAILING_DECIMAL_POINT flag.
- // Example: 2345.0 is converted to "2345.0".
- // - UNIQUE_ZERO: "-0.0" is converted to "0.0".
- //
- // Infinity symbol and nan_symbol provide the string representation for these
- // special values. If the string is NULL and the special value is encountered
- // then the conversion functions return false.
- //
- // The exponent_character is used in exponential representations. It is
- // usually 'e' or 'E'.
- //
- // When converting to the shortest representation the converter will
- // represent input numbers in decimal format if they are in the interval
- // [10^decimal_in_shortest_low; 10^decimal_in_shortest_high[
- // (lower boundary included, greater boundary excluded).
- // Example: with decimal_in_shortest_low = -6 and
- // decimal_in_shortest_high = 21:
- // ToShortest(0.000001) -> "0.000001"
- // ToShortest(0.0000001) -> "1e-7"
- // ToShortest(111111111111111111111.0) -> "111111111111111110000"
- // ToShortest(100000000000000000000.0) -> "100000000000000000000"
- // ToShortest(1111111111111111111111.0) -> "1.1111111111111111e+21"
- //
- // When converting to precision mode the converter may add
- // max_leading_padding_zeroes before returning the number in exponential
- // format.
- // Example with max_leading_padding_zeroes_in_precision_mode = 6.
- // ToPrecision(0.0000012345, 2) -> "0.0000012"
- // ToPrecision(0.00000012345, 2) -> "1.2e-7"
- // Similarily the converter may add up to
- // max_trailing_padding_zeroes_in_precision_mode in precision mode to avoid
- // returning an exponential representation. A zero added by the
- // EMIT_TRAILING_ZERO_AFTER_POINT flag is counted for this limit.
- // Examples for max_trailing_padding_zeroes_in_precision_mode = 1:
- // ToPrecision(230.0, 2) -> "230"
- // ToPrecision(230.0, 2) -> "230." with EMIT_TRAILING_DECIMAL_POINT.
- // ToPrecision(230.0, 2) -> "2.3e2" with EMIT_TRAILING_ZERO_AFTER_POINT.
- DoubleToStringConverter(int flags,
- const char* infinity_symbol,
- const char* nan_symbol,
- char exponent_character,
- int decimal_in_shortest_low,
- int decimal_in_shortest_high,
- int max_leading_padding_zeroes_in_precision_mode,
- int max_trailing_padding_zeroes_in_precision_mode)
- : flags_(flags),
- infinity_symbol_(infinity_symbol),
- nan_symbol_(nan_symbol),
- exponent_character_(exponent_character),
- decimal_in_shortest_low_(decimal_in_shortest_low),
- decimal_in_shortest_high_(decimal_in_shortest_high),
- max_leading_padding_zeroes_in_precision_mode_(
- max_leading_padding_zeroes_in_precision_mode),
- max_trailing_padding_zeroes_in_precision_mode_(
- max_trailing_padding_zeroes_in_precision_mode) {
- // When 'trailing zero after the point' is set, then 'trailing point'
- // must be set too.
- ASSERT(((flags & EMIT_TRAILING_DECIMAL_POINT) != 0) ||
- !((flags & EMIT_TRAILING_ZERO_AFTER_POINT) != 0));
- }
-
- // Returns a converter following the EcmaScript specification.
- static const DoubleToStringConverter& EcmaScriptConverter();
-
- // Computes the shortest string of digits that correctly represent the input
- // number. Depending on decimal_in_shortest_low and decimal_in_shortest_high
- // (see constructor) it then either returns a decimal representation, or an
- // exponential representation.
- // Example with decimal_in_shortest_low = -6,
- // decimal_in_shortest_high = 21,
- // EMIT_POSITIVE_EXPONENT_SIGN activated, and
- // EMIT_TRAILING_DECIMAL_POINT deactived:
- // ToShortest(0.000001) -> "0.000001"
- // ToShortest(0.0000001) -> "1e-7"
- // ToShortest(111111111111111111111.0) -> "111111111111111110000"
- // ToShortest(100000000000000000000.0) -> "100000000000000000000"
- // ToShortest(1111111111111111111111.0) -> "1.1111111111111111e+21"
- //
- // Note: the conversion may round the output if the returned string
- // is accurate enough to uniquely identify the input-number.
- // For example the most precise representation of the double 9e59 equals
- // "899999999999999918767229449717619953810131273674690656206848", but
- // the converter will return the shorter (but still correct) "9e59".
- //
- // Returns true if the conversion succeeds. The conversion always succeeds
- // except when the input value is special and no infinity_symbol or
- // nan_symbol has been given to the constructor.
- bool ToShortest(double value, StringBuilder* result_builder) const {
- return ToShortestIeeeNumber(value, result_builder, SHORTEST);
- }
-
- // Same as ToShortest, but for single-precision floats.
- bool ToShortestSingle(float value, StringBuilder* result_builder) const {
- return ToShortestIeeeNumber(value, result_builder, SHORTEST_SINGLE);
- }
-
-
- // Computes a decimal representation with a fixed number of digits after the
- // decimal point. The last emitted digit is rounded.
- //
- // Examples:
- // ToFixed(3.12, 1) -> "3.1"
- // ToFixed(3.1415, 3) -> "3.142"
- // ToFixed(1234.56789, 4) -> "1234.5679"
- // ToFixed(1.23, 5) -> "1.23000"
- // ToFixed(0.1, 4) -> "0.1000"
- // ToFixed(1e30, 2) -> "1000000000000000019884624838656.00"
- // ToFixed(0.1, 30) -> "0.100000000000000005551115123126"
- // ToFixed(0.1, 17) -> "0.10000000000000001"
- //
- // If requested_digits equals 0, then the tail of the result depends on
- // the EMIT_TRAILING_DECIMAL_POINT and EMIT_TRAILING_ZERO_AFTER_POINT.
- // Examples, for requested_digits == 0,
- // let EMIT_TRAILING_DECIMAL_POINT and EMIT_TRAILING_ZERO_AFTER_POINT be
- // - false and false: then 123.45 -> 123
- // 0.678 -> 1
- // - true and false: then 123.45 -> 123.
- // 0.678 -> 1.
- // - true and true: then 123.45 -> 123.0
- // 0.678 -> 1.0
- //
- // Returns true if the conversion succeeds. The conversion always succeeds
- // except for the following cases:
- // - the input value is special and no infinity_symbol or nan_symbol has
- // been provided to the constructor,
- // - 'value' > 10^kMaxFixedDigitsBeforePoint, or
- // - 'requested_digits' > kMaxFixedDigitsAfterPoint.
- // The last two conditions imply that the result will never contain more than
- // 1 + kMaxFixedDigitsBeforePoint + 1 + kMaxFixedDigitsAfterPoint characters
- // (one additional character for the sign, and one for the decimal point).
- bool ToFixed(double value,
- int requested_digits,
- StringBuilder* result_builder) const;
-
- // Computes a representation in exponential format with requested_digits
- // after the decimal point. The last emitted digit is rounded.
- // If requested_digits equals -1, then the shortest exponential representation
- // is computed.
- //
- // Examples with EMIT_POSITIVE_EXPONENT_SIGN deactivated, and
- // exponent_character set to 'e'.
- // ToExponential(3.12, 1) -> "3.1e0"
- // ToExponential(5.0, 3) -> "5.000e0"
- // ToExponential(0.001, 2) -> "1.00e-3"
- // ToExponential(3.1415, -1) -> "3.1415e0"
- // ToExponential(3.1415, 4) -> "3.1415e0"
- // ToExponential(3.1415, 3) -> "3.142e0"
- // ToExponential(123456789000000, 3) -> "1.235e14"
- // ToExponential(1000000000000000019884624838656.0, -1) -> "1e30"
- // ToExponential(1000000000000000019884624838656.0, 32) ->
- // "1.00000000000000001988462483865600e30"
- // ToExponential(1234, 0) -> "1e3"
- //
- // Returns true if the conversion succeeds. The conversion always succeeds
- // except for the following cases:
- // - the input value is special and no infinity_symbol or nan_symbol has
- // been provided to the constructor,
- // - 'requested_digits' > kMaxExponentialDigits.
- // The last condition implies that the result will never contain more than
- // kMaxExponentialDigits + 8 characters (the sign, the digit before the
- // decimal point, the decimal point, the exponent character, the
- // exponent's sign, and at most 3 exponent digits).
- bool ToExponential(double value,
- int requested_digits,
- StringBuilder* result_builder) const;
-
- // Computes 'precision' leading digits of the given 'value' and returns them
- // either in exponential or decimal format, depending on
- // max_{leading|trailing}_padding_zeroes_in_precision_mode (given to the
- // constructor).
- // The last computed digit is rounded.
- //
- // Example with max_leading_padding_zeroes_in_precision_mode = 6.
- // ToPrecision(0.0000012345, 2) -> "0.0000012"
- // ToPrecision(0.00000012345, 2) -> "1.2e-7"
- // Similarily the converter may add up to
- // max_trailing_padding_zeroes_in_precision_mode in precision mode to avoid
- // returning an exponential representation. A zero added by the
- // EMIT_TRAILING_ZERO_AFTER_POINT flag is counted for this limit.
- // Examples for max_trailing_padding_zeroes_in_precision_mode = 1:
- // ToPrecision(230.0, 2) -> "230"
- // ToPrecision(230.0, 2) -> "230." with EMIT_TRAILING_DECIMAL_POINT.
- // ToPrecision(230.0, 2) -> "2.3e2" with EMIT_TRAILING_ZERO_AFTER_POINT.
- // Examples for max_trailing_padding_zeroes_in_precision_mode = 3, and no
- // EMIT_TRAILING_ZERO_AFTER_POINT:
- // ToPrecision(123450.0, 6) -> "123450"
- // ToPrecision(123450.0, 5) -> "123450"
- // ToPrecision(123450.0, 4) -> "123500"
- // ToPrecision(123450.0, 3) -> "123000"
- // ToPrecision(123450.0, 2) -> "1.2e5"
- //
- // Returns true if the conversion succeeds. The conversion always succeeds
- // except for the following cases:
- // - the input value is special and no infinity_symbol or nan_symbol has
- // been provided to the constructor,
- // - precision < kMinPericisionDigits
- // - precision > kMaxPrecisionDigits
- // The last condition implies that the result will never contain more than
- // kMaxPrecisionDigits + 7 characters (the sign, the decimal point, the
- // exponent character, the exponent's sign, and at most 3 exponent digits).
- bool ToPrecision(double value,
- int precision,
- StringBuilder* result_builder) const;
-
- enum DtoaMode {
- // Produce the shortest correct representation.
- // For example the output of 0.299999999999999988897 is (the less accurate
- // but correct) 0.3.
- SHORTEST,
- // Same as SHORTEST, but for single-precision floats.
- SHORTEST_SINGLE,
- // Produce a fixed number of digits after the decimal point.
- // For instance fixed(0.1, 4) becomes 0.1000
- // If the input number is big, the output will be big.
- FIXED,
- // Fixed number of digits (independent of the decimal point).
- PRECISION
- };
-
- // The maximal number of digits that are needed to emit a double in base 10.
- // A higher precision can be achieved by using more digits, but the shortest
- // accurate representation of any double will never use more digits than
- // kBase10MaximalLength.
- // Note that DoubleToAscii null-terminates its input. So the given buffer
- // should be at least kBase10MaximalLength + 1 characters long.
- static const int kBase10MaximalLength = 17;
-
- // Converts the given double 'v' to ascii. 'v' must not be NaN, +Infinity, or
- // -Infinity. In SHORTEST_SINGLE-mode this restriction also applies to 'v'
- // after it has been casted to a single-precision float. That is, in this
- // mode static_cast<float>(v) must not be NaN, +Infinity or -Infinity.
- //
- // The result should be interpreted as buffer * 10^(point-length).
- //
- // The output depends on the given mode:
- // - SHORTEST: produce the least amount of digits for which the internal
- // identity requirement is still satisfied. If the digits are printed
- // (together with the correct exponent) then reading this number will give
- // 'v' again. The buffer will choose the representation that is closest to
- // 'v'. If there are two at the same distance, than the one farther away
- // from 0 is chosen (halfway cases - ending with 5 - are rounded up).
- // In this mode the 'requested_digits' parameter is ignored.
- // - SHORTEST_SINGLE: same as SHORTEST but with single-precision.
- // - FIXED: produces digits necessary to print a given number with
- // 'requested_digits' digits after the decimal point. The produced digits
- // might be too short in which case the caller has to fill the remainder
- // with '0's.
- // Example: toFixed(0.001, 5) is allowed to return buffer="1", point=-2.
- // Halfway cases are rounded towards +/-Infinity (away from 0). The call
- // toFixed(0.15, 2) thus returns buffer="2", point=0.
- // The returned buffer may contain digits that would be truncated from the
- // shortest representation of the input.
- // - PRECISION: produces 'requested_digits' where the first digit is not '0'.
- // Even though the length of produced digits usually equals
- // 'requested_digits', the function is allowed to return fewer digits, in
- // which case the caller has to fill the missing digits with '0's.
- // Halfway cases are again rounded away from 0.
- // DoubleToAscii expects the given buffer to be big enough to hold all
- // digits and a terminating null-character. In SHORTEST-mode it expects a
- // buffer of at least kBase10MaximalLength + 1. In all other modes the
- // requested_digits parameter and the padding-zeroes limit the size of the
- // output. Don't forget the decimal point, the exponent character and the
- // terminating null-character when computing the maximal output size.
- // The given length is only used in debug mode to ensure the buffer is big
- // enough.
- static void DoubleToAscii(double v,
- DtoaMode mode,
- int requested_digits,
- char* buffer,
- int buffer_length,
- bool* sign,
- int* length,
- int* point);
-
- private:
- // Implementation for ToShortest and ToShortestSingle.
- bool ToShortestIeeeNumber(double value,
- StringBuilder* result_builder,
- DtoaMode mode) const;
-
- // If the value is a special value (NaN or Infinity) constructs the
- // corresponding string using the configured infinity/nan-symbol.
- // If either of them is NULL or the value is not special then the
- // function returns false.
- bool HandleSpecialValues(double value, StringBuilder* result_builder) const;
- // Constructs an exponential representation (i.e. 1.234e56).
- // The given exponent assumes a decimal point after the first decimal digit.
- void CreateExponentialRepresentation(const char* decimal_digits,
- int length,
- int exponent,
- StringBuilder* result_builder) const;
- // Creates a decimal representation (i.e 1234.5678).
- void CreateDecimalRepresentation(const char* decimal_digits,
- int length,
- int decimal_point,
- int digits_after_point,
- StringBuilder* result_builder) const;
-
- const int flags_;
- const char* const infinity_symbol_;
- const char* const nan_symbol_;
- const char exponent_character_;
- const int decimal_in_shortest_low_;
- const int decimal_in_shortest_high_;
- const int max_leading_padding_zeroes_in_precision_mode_;
- const int max_trailing_padding_zeroes_in_precision_mode_;
-
- DISALLOW_IMPLICIT_CONSTRUCTORS(DoubleToStringConverter);
-};
-
-
-class StringToDoubleConverter {
- public:
- // Enumeration for allowing octals and ignoring junk when converting
- // strings to numbers.
- enum Flags {
- NO_FLAGS = 0,
- ALLOW_HEX = 1,
- ALLOW_OCTALS = 2,
- ALLOW_TRAILING_JUNK = 4,
- ALLOW_LEADING_SPACES = 8,
- ALLOW_TRAILING_SPACES = 16,
- ALLOW_SPACES_AFTER_SIGN = 32
- };
-
- // Flags should be a bit-or combination of the possible Flags-enum.
- // - NO_FLAGS: no special flags.
- // - ALLOW_HEX: recognizes the prefix "0x". Hex numbers may only be integers.
- // Ex: StringToDouble("0x1234") -> 4660.0
- // In StringToDouble("0x1234.56") the characters ".56" are trailing
- // junk. The result of the call is hence dependent on
- // the ALLOW_TRAILING_JUNK flag and/or the junk value.
- // With this flag "0x" is a junk-string. Even with ALLOW_TRAILING_JUNK,
- // the string will not be parsed as "0" followed by junk.
- //
- // - ALLOW_OCTALS: recognizes the prefix "0" for octals:
- // If a sequence of octal digits starts with '0', then the number is
- // read as octal integer. Octal numbers may only be integers.
- // Ex: StringToDouble("01234") -> 668.0
- // StringToDouble("012349") -> 12349.0 // Not a sequence of octal
- // // digits.
- // In StringToDouble("01234.56") the characters ".56" are trailing
- // junk. The result of the call is hence dependent on
- // the ALLOW_TRAILING_JUNK flag and/or the junk value.
- // In StringToDouble("01234e56") the characters "e56" are trailing
- // junk, too.
- // - ALLOW_TRAILING_JUNK: ignore trailing characters that are not part of
- // a double literal.
- // - ALLOW_LEADING_SPACES: skip over leading spaces.
- // - ALLOW_TRAILING_SPACES: ignore trailing spaces.
- // - ALLOW_SPACES_AFTER_SIGN: ignore spaces after the sign.
- // Ex: StringToDouble("- 123.2") -> -123.2.
- // StringToDouble("+ 123.2") -> 123.2
- //
- // empty_string_value is returned when an empty string is given as input.
- // If ALLOW_LEADING_SPACES or ALLOW_TRAILING_SPACES are set, then a string
- // containing only spaces is converted to the 'empty_string_value', too.
- //
- // junk_string_value is returned when
- // a) ALLOW_TRAILING_JUNK is not set, and a junk character (a character not
- // part of a double-literal) is found.
- // b) ALLOW_TRAILING_JUNK is set, but the string does not start with a
- // double literal.
- //
- // infinity_symbol and nan_symbol are strings that are used to detect
- // inputs that represent infinity and NaN. They can be null, in which case
- // they are ignored.
- // The conversion routine first reads any possible signs. Then it compares the
- // following character of the input-string with the first character of
- // the infinity, and nan-symbol. If either matches, the function assumes, that
- // a match has been found, and expects the following input characters to match
- // the remaining characters of the special-value symbol.
- // This means that the following restrictions apply to special-value symbols:
- // - they must not start with signs ('+', or '-'),
- // - they must not have the same first character.
- // - they must not start with digits.
- //
- // Examples:
- // flags = ALLOW_HEX | ALLOW_TRAILING_JUNK,
- // empty_string_value = 0.0,
- // junk_string_value = NaN,
- // infinity_symbol = "infinity",
- // nan_symbol = "nan":
- // StringToDouble("0x1234") -> 4660.0.
- // StringToDouble("0x1234K") -> 4660.0.
- // StringToDouble("") -> 0.0 // empty_string_value.
- // StringToDouble(" ") -> NaN // junk_string_value.
- // StringToDouble(" 1") -> NaN // junk_string_value.
- // StringToDouble("0x") -> NaN // junk_string_value.
- // StringToDouble("-123.45") -> -123.45.
- // StringToDouble("--123.45") -> NaN // junk_string_value.
- // StringToDouble("123e45") -> 123e45.
- // StringToDouble("123E45") -> 123e45.
- // StringToDouble("123e+45") -> 123e45.
- // StringToDouble("123E-45") -> 123e-45.
- // StringToDouble("123e") -> 123.0 // trailing junk ignored.
- // StringToDouble("123e-") -> 123.0 // trailing junk ignored.
- // StringToDouble("+NaN") -> NaN // NaN string literal.
- // StringToDouble("-infinity") -> -inf. // infinity literal.
- // StringToDouble("Infinity") -> NaN // junk_string_value.
- //
- // flags = ALLOW_OCTAL | ALLOW_LEADING_SPACES,
- // empty_string_value = 0.0,
- // junk_string_value = NaN,
- // infinity_symbol = NULL,
- // nan_symbol = NULL:
- // StringToDouble("0x1234") -> NaN // junk_string_value.
- // StringToDouble("01234") -> 668.0.
- // StringToDouble("") -> 0.0 // empty_string_value.
- // StringToDouble(" ") -> 0.0 // empty_string_value.
- // StringToDouble(" 1") -> 1.0
- // StringToDouble("0x") -> NaN // junk_string_value.
- // StringToDouble("0123e45") -> NaN // junk_string_value.
- // StringToDouble("01239E45") -> 1239e45.
- // StringToDouble("-infinity") -> NaN // junk_string_value.
- // StringToDouble("NaN") -> NaN // junk_string_value.
- StringToDoubleConverter(int flags,
- double empty_string_value,
- double junk_string_value,
- const char* infinity_symbol,
- const char* nan_symbol)
- : flags_(flags),
- empty_string_value_(empty_string_value),
- junk_string_value_(junk_string_value),
- infinity_symbol_(infinity_symbol),
- nan_symbol_(nan_symbol) {
- }
-
- // Performs the conversion.
- // The output parameter 'processed_characters_count' is set to the number
- // of characters that have been processed to read the number.
- // Spaces than are processed with ALLOW_{LEADING|TRAILING}_SPACES are included
- // in the 'processed_characters_count'. Trailing junk is never included.
- double StringToDouble(const char* buffer,
- int length,
- int* processed_characters_count) const {
- return StringToIeee(buffer, length, processed_characters_count, true);
- }
-
- // Same as StringToDouble but reads a float.
- // Note that this is not equivalent to static_cast<float>(StringToDouble(...))
- // due to potential double-rounding.
- float StringToFloat(const char* buffer,
- int length,
- int* processed_characters_count) const {
- return static_cast<float>(StringToIeee(buffer, length,
- processed_characters_count, false));
- }
-
- private:
- const int flags_;
- const double empty_string_value_;
- const double junk_string_value_;
- const char* const infinity_symbol_;
- const char* const nan_symbol_;
-
- double StringToIeee(const char* buffer,
- int length,
- int* processed_characters_count,
- bool read_as_double) const;
-
- DISALLOW_IMPLICIT_CONSTRUCTORS(StringToDoubleConverter);
-};
-
-} // namespace double_conversion
-
-#endif // DOUBLE_CONVERSION_DOUBLE_CONVERSION_H_
[40/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/como-linux.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/como-linux.jam b/ext/kenlm/jam-files/boost-build/tools/como-linux.jam
deleted file mode 100644
index 5c554c8..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/como-linux.jam
+++ /dev/null
@@ -1,103 +0,0 @@
-# Copyright 2004, 2005, 2006 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-# The following #// line will be used by the regression test table generation
-# program as the column heading for HTML tables. Must not include a version
-# number.
-#//<a href="http://www.comeaucomputing.com/">Comeau<br>C++</a>
-
-import toolset ;
-import feature ;
-import toolset : flags ;
-import common ;
-import generators ;
-
-import unix ;
-import como ;
-
-feature.extend-subfeature toolset como : platform : linux ;
-
-toolset.inherit-generators como-linux
- <toolset>como <toolset-como:platform>linux : unix ;
-generators.override como-linux.prebuilt : builtin.lib-generator ;
-generators.override como-linux.searched-lib-generator : searched-lib-generator ;
-toolset.inherit-flags como-linux : unix ;
-toolset.inherit-rules como-linux : gcc ;
-
-generators.register-c-compiler como-linux.compile.c++ : CPP : OBJ
- : <toolset>como <toolset-como:platform>linux ;
-generators.register-c-compiler como-linux.compile.c : C : OBJ
- : <toolset>como <toolset-como:platform>linux ;
-
-
-rule init ( version ? : command * : options * )
-{
- local condition = [ common.check-init-parameters como-linux
- : version $(version) ] ;
-
- command = [ common.get-invocation-command como-linux : como
- : $(command) ] ;
-
- common.handle-options como-linux : $(condition) : $(command) : $(options) ;
-}
-
-
-flags como-linux C++FLAGS <exception-handling>off : --no_exceptions ;
-flags como-linux C++FLAGS <exception-handling>on : --exceptions ;
-
-flags como-linux CFLAGS <inlining>off : --no_inlining ;
-flags como-linux CFLAGS <inlining>on <inlining>full : --inlining ;
-
-flags como-linux CFLAGS <optimization>off : -O0 ;
-flags como-linux CFLAGS <optimization>speed : -O3 ;
-flags como-linux CFLAGS <optimization>space : -Os ;
-
-flags como-linux CFLAGS <debug-symbols>on : -g ;
-flags como-linux LINKFLAGS <debug-symbols>on : -g ;
-
-flags como-linux FINDLIBS : m ;
-flags como-linux FINDLIBS : rt ;
-
-flags como-linux CFLAGS <cflags> ;
-flags como-linux C++FLAGS <cxxflags> ;
-flags como-linux DEFINES <define> ;
-flags como-linux UNDEFS <undef> ;
-flags como-linux HDRS <include> ;
-flags como-linux STDHDRS <sysinclude> ;
-flags como-linux LINKFLAGS <linkflags> ;
-flags como-linux ARFLAGS <arflags> ;
-
-flags como-linux.link LIBRARIES <library-file> ;
-flags como-linux.link LINKPATH <library-path> ;
-flags como-linux.link FINDLIBS-ST <find-static-library> ;
-flags como-linux.link FINDLIBS-SA <find-shared-library> ;
-
-flags como-linux.link RPATH <dll-path> ;
-flags como-linux.link RPATH_LINK <xdll-path> ;
-
-
-actions link bind LIBRARIES
-{
- $(CONFIG_COMMAND) $(LINKFLAGS) -o "$(<[1])" "$(>)" -L"$(LINKPATH)" -Wl,-R$(SPACE)-Wl,"$(RPATH)" -Wl,-rpath-link$(SPACE)-Wl,"$(RPATH_LINK)" "$(LIBRARIES)" "$(LIBRARIES)" -l$(FINDLIBS-SA) -l$(FINDLIBS-ST) 2>&1
-}
-
-actions link.dll bind LIBRARIES
-{
- $(CONFIG_COMMAND) $(LINKFLAGS) -shared -o "$(<[1])" "$(>)" -L"$(LINKPATH)" -Wl,-R$(SPACE)-Wl,"$(RPATH)" -Wl,-rpath-link$(SPACE)-Wl,"$(RPATH_LINK)" "$(LIBRARIES)" "$(LIBRARIES)" -l$(FINDLIBS-SA) -l$(FINDLIBS-ST) 2>&1
-}
-
-actions compile.c
-{
- $(CONFIG_COMMAND) -c --c99 --long_long -U$(UNDEFS) -D$(DEFINES) $(CFLAGS) -I"$(HDRS)" -I"$(STDHDRS)" -o "$(<)" "$(>)" 2>&1
-}
-
-actions compile.c++
-{
- $(CONFIG_COMMAND) -tused -c --long_long -U$(UNDEFS) -D$(DEFINES) $(CFLAGS) $(C++FLAGS) -I"$(HDRS)" -I"$(STDHDRS)" -o "$(<)" "$(>)" 2>&1
-}
-
-actions archive
-{
- ar rcu $(<) $(>)
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/como-win.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/como-win.jam b/ext/kenlm/jam-files/boost-build/tools/como-win.jam
deleted file mode 100644
index d21a70d..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/como-win.jam
+++ /dev/null
@@ -1,117 +0,0 @@
-# (C) Copyright David Abrahams 2001.
-# (C) Copyright MetaCommunications, Inc. 2004.
-
-# Distributed under the Boost Software License, Version 1.0. (See
-# accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# The following #// line will be used by the regression test table generation
-# program as the column heading for HTML tables. Must not include a version
-# number.
-#//<a href="http://www.comeaucomputing.com/">Comeau<br>C++</a>
-
-import common ;
-import como ;
-import feature ;
-import generators ;
-import toolset : flags ;
-
-feature.extend-subfeature toolset como : platform : win ;
-
-
-# Initializes the Comeau toolset for windows. The command is the command which
-# invokes the compiler. You should either set environment variable
-# COMO_XXX_INCLUDE where XXX is the used backend (as described in the
-# documentation), or pass that as part of command, e.g:
-#
-# using como-win : 4.3 : "set COMO_BCC_INCLUDE=C:/include &&" como.exe ;
-#
-rule init ( version ? : command * : options * )
-{
- local condition = [ common.check-init-parameters como-win
- : version $(version) ] ;
-
- command = [ common.get-invocation-command como-win : como.exe :
- $(command) ] ;
-
- common.handle-options como-win : $(condition) : $(command) : $(options) ;
-}
-
-generators.register-c-compiler como-win.compile.c++ : CPP : OBJ
- : <toolset>como <toolset-como:platform>win ;
-generators.register-c-compiler como-win.compile.c : C : OBJ
- : <toolset>como <toolset-como:platform>win ;
-
-
-generators.register-linker como-win.link
- : OBJ SEARCHED_LIB STATIC_LIB IMPORT_LIB
- : EXE
- : <toolset>como <toolset-como:platform>win ;
-
-# Note that status of shared libraries support is not clear, so we do not define
-# the link.dll generator.
-generators.register-archiver como-win.archive
- : OBJ : STATIC_LIB
- : <toolset>como <toolset-como:platform>win ;
-
-
-flags como-win C++FLAGS <exception-handling>off : --no_exceptions ;
-flags como-win C++FLAGS <exception-handling>on : --exceptions ;
-
-flags como-win CFLAGS <inlining>off : --no_inlining ;
-flags como-win CFLAGS <inlining>on <inlining>full : --inlining ;
-
-
-# The following seems to be VC-specific options. At least, when I uncomment
-# then, Comeau with bcc as backend reports that bcc32 invocation failed.
-#
-#flags como-win CFLAGS <debug-symbols>on : /Zi ;
-#flags como-win CFLAGS <optimization>off : /Od ;
-
-
-flags como-win CFLAGS <cflags> ;
-flags como-win CFLAGS : -D_WIN32 ; # Make sure that we get the Boost Win32 platform config header.
-flags como-win CFLAGS <threading>multi : -D_MT ; # Make sure that our config knows that threading is on.
-flags como-win C++FLAGS <cxxflags> ;
-flags como-win DEFINES <define> ;
-flags como-win UNDEFS <undef> ;
-flags como-win HDRS <include> ;
-flags como-win SYSHDRS <sysinclude> ;
-flags como-win LINKFLAGS <linkflags> ;
-flags como-win ARFLAGS <arflags> ;
-flags como-win NO_WARN <no-warn> ;
-
-#flags como-win STDHDRS : $(COMO_INCLUDE_PATH) ;
-#flags como-win STDLIB_PATH : $(COMO_STDLIB_PATH)$(SLASH) ;
-
-flags como-win LIBPATH <library-path> ;
-flags como-win LIBRARIES <library-file> ;
-flags como-win FINDLIBS <find-shared-library> ;
-flags como-win FINDLIBS <find-static-library> ;
-
-nl = "
-" ;
-
-
-# For como, we repeat all libraries so that dependencies are always resolved.
-#
-actions link bind LIBRARIES
-{
- $(CONFIG_COMMAND) --no_version --no_prelink_verbose $(LINKFLAGS) -o "$(<[1]:S=)" @"@($(<[1]:W).rsp:E=$(nl)"$(>)")" "$(LIBRARIES)" "$(FINDLIBS:S=.lib)"
-}
-
-actions compile.c
-{
- $(CONFIG_COMMAND) -c --c99 -e5 --no_version --display_error_number --diag_suppress=9,21,161,748,940,962 -U$(UNDEFS) -D$(DEFINES) $(WARN) $(CFLAGS) -I"$(HDRS)" -I"$(STDHDRS)" -I"$(SYSHDRS)" -o "$(<:D=)" "$(>)"
-}
-
-actions compile.c++
-{
- $(CONFIG_COMMAND) -c -e5 --no_version --no_prelink_verbose --display_error_number --long_long --diag_suppress=9,21,161,748,940,962 --diag_error=461 -D__STL_LONG_LONG -U$(UNDEFS) -D$(DEFINES) $(WARN) $(CFLAGS) $(C++FLAGS) -I"$(HDRS)" -I"$(STDHDRS)" -I"$(SYSHDRS)" -o "$(<)" "$(>)"
-}
-
-actions archive
-{
- $(CONFIG_COMMAND) --no_version --no_prelink_verbose --prelink_object @"@($(<[1]:W).rsp:E=$(nl)"$(>)")"
- lib $(ARFLAGS) /nologo /out:"$(<:S=.lib)" @"@($(<[1]:W).rsp:E=$(nl)"$(>)")"
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/como.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/como.jam b/ext/kenlm/jam-files/boost-build/tools/como.jam
deleted file mode 100644
index 04a05a9..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/como.jam
+++ /dev/null
@@ -1,29 +0,0 @@
-# Copyright Vladimir Prus 2004.
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt
-# or copy at http://www.boost.org/LICENSE_1_0.txt)
-
-# This is a generic 'como' toolset. Depending on the current system, it
-# forwards either to 'como-linux' or 'como-win' modules.
-
-import feature ;
-import os ;
-import toolset ;
-
-feature.extend toolset : como ;
-feature.subfeature toolset como : platform : : propagated link-incompatible ;
-
-rule init ( * : * )
-{
- if [ os.name ] = LINUX
- {
- toolset.using como-linux :
- $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ;
- }
- else
- {
- toolset.using como-win :
- $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ;
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/convert.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/convert.jam b/ext/kenlm/jam-files/boost-build/tools/convert.jam
deleted file mode 100644
index ac1d701..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/convert.jam
+++ /dev/null
@@ -1,62 +0,0 @@
-# Copyright (c) 2009 Vladimir Prus
-#
-# Use, modification and distribution is subject to the Boost Software
-# License Version 1.0. (See accompanying file LICENSE_1_0.txt or
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# Implements 'convert' target that takes a bunch of source and
-# tries to convert each one to the specified type.
-#
-# For example:
-#
-# convert objects obj : a.cpp b.cpp ;
-#
-
-import targets ;
-import generators ;
-import project ;
-import type ;
-import "class" : new ;
-
-class convert-target-class : typed-target
-{
- rule __init__ ( name : project : type
- : sources * : requirements * : default-build * : usage-requirements * )
- {
- typed-target.__init__ $(name) : $(project) : $(type)
- : $(sources) : $(requirements) : $(default-build) : $(usage-requirements) ;
- }
-
- rule construct ( name : source-targets * : property-set )
- {
- local r = [ generators.construct $(self.project) : $(self.type)
- : [ property-set.create [ $(property-set).raw ] # [ feature.expand
- <main-target-type>$(self.type) ]
- # ]
- : $(source-targets) ] ;
- if ! $(r)
- {
- errors.error "unable to construct" [ full-name ] ;
- }
-
- return $(r) ;
- }
-
-}
-
-rule convert ( name type : sources * : requirements * : default-build *
- : usage-requirements * )
-{
- local project = [ project.current ] ;
-
- # This is a circular module dependency, so it must be imported here
- modules.import targets ;
- targets.main-target-alternative
- [ new convert-target-class $(name) : $(project) : [ type.type-from-rule-name $(type) ]
- : [ targets.main-target-sources $(sources) : $(name) ]
- : [ targets.main-target-requirements $(requirements) : $(project) ]
- : [ targets.main-target-default-build $(default-build) : $(project) ]
- : [ targets.main-target-usage-requirements $(usage-requirements) : $(project) ]
- ] ;
-}
-IMPORT $(__name__) : convert : : convert ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/cray.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/cray.jam b/ext/kenlm/jam-files/boost-build/tools/cray.jam
deleted file mode 100644
index a37fe23..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/cray.jam
+++ /dev/null
@@ -1,115 +0,0 @@
-# Copyright 2001 David Abrahams.
-# Copyright 2004, 2005 Markus Schoepflin.
-# Copyright 2011, John Maddock
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-#
-# Cray C++ Compiler
-# See http://docs.cray.com/books/S-2179-50/html-S-2179-50/S-2179-50-toc.html
-#
-
-import feature generators common ;
-import toolset : flags ;
-
-feature.extend toolset : cray ;
-
-# Inherit from Unix toolset to get library ordering magic.
-toolset.inherit cray : unix ;
-
-generators.override cray.prebuilt : builtin.lib-generator ;
-generators.override cray.prebuilt : builtin.prebuilt ;
-generators.override cray.searched-lib-generator : searched-lib-generator ;
-
-
-rule init ( version ? : command * : options * )
-{
- local condition = [ common.check-init-parameters cray : version $(version) ] ;
-
- local command = [ common.get-invocation-command cray : CC : $(command) ] ;
-
- if $(command)
- {
- local root = [ common.get-absolute-tool-path $(command[-1]) ] ;
-
- if $(root)
- {
- flags cray .root $(condition) : "\"$(root)\"/" ;
- }
- }
- # If we can't find 'CC' anyway, at least show 'CC' in the commands
- command ?= CC ;
-
- common.handle-options cray : $(condition) : $(command) : $(options) ;
-}
-
-generators.register-c-compiler cray.compile.c++ : CPP : OBJ : <toolset>cray ;
-generators.register-c-compiler cray.compile.c : C : OBJ : <toolset>cray ;
-
-
-
-# No static linking as far as I can tell.
-# flags cxx LINKFLAGS <runtime-link>static : -bstatic ;
-flags cray.compile OPTIONS <debug-symbols>on : -Gn ;
-flags cray.link OPTIONS <debug-symbols>on : -Gn ;
-
-flags cray.compile OPTIONS <optimization>off : -O0 ;
-flags cray.compile OPTIONS <optimization>speed : -O3 ;
-flags cray.compile OPTIONS <optimization>space : -O1 ;
-
-flags cray.compile OPTIONS <cflags> ;
-flags cray.compile.c++ OPTIONS <cxxflags> ;
-flags cray.compile DEFINES <define> ;
-flags cray.compile INCLUDES <include> ;
-flags cray.link OPTIONS <linkflags> ;
-
-flags cray.compile OPTIONS <link>shared : -fPIC ;
-flags cray.link OPTIONS <link>shared : -fPIC ;
-
-flags cray.link LIBPATH <library-path> ;
-flags cray.link LIBRARIES <library-file> ;
-flags cray.link FINDLIBS-ST <find-static-library> ;
-flags cray.link FINDLIBS-SA <find-shared-library> ;
-
-actions link bind LIBRARIES
-{
- $(CONFIG_COMMAND) $(OPTIONS) -o "$(<)" -L$(LIBPATH) "$(>)" "$(LIBRARIES)" -l$(FINDLIBS-ST) -l$(FINDLIBS-SA)
-}
-
-# When creating dynamic libraries, we don't want to be warned about unresolved
-# symbols, therefore all unresolved symbols are marked as expected by
-# '-expect_unresolved *'. This also mirrors the behaviour of the GNU tool
-# chain.
-
-actions link.dll bind LIBRARIES
-{
- $(CONFIG_COMMAND) -shared $(OPTIONS) -o "$(<[1])" -L$(LIBPATH) "$(>)" "$(LIBRARIES)" -l$(FINDLIBS-ST) -l$(FINDLIBS-SA)
-}
-
-
-# Note: Relaxed ANSI mode (-std) is used for compilation because in strict ANSI
-# C89 mode (-std1) the compiler doesn't accept C++ comments in C files. As -std
-# is the default, no special flag is needed.
-actions compile.c
-{
- $(.root:E=)cc -c $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -o "$(<)" "$(>)"
-}
-
-# Note: The compiler is forced to compile the files as C++ (-x cxx) because
-# otherwise it will silently ignore files with no file extension.
-#
-# Note: We deliberately don't suppress any warnings on the compiler command
-# line, the user can always do this in a customized toolset later on.
-
-actions compile.c++
-{
- $(CONFIG_COMMAND) -c -h gnu $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -o "$(<)" "$(>)"
-}
-
-# Always create archive from scratch. See the gcc toolet for rationale.
-RM = [ common.rm-command ] ;
-actions together piecemeal archive
-{
- $(RM) "$(<)"
- ar rc $(<) $(>)
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/cw-config.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/cw-config.jam b/ext/kenlm/jam-files/boost-build/tools/cw-config.jam
deleted file mode 100644
index 1211b7c..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/cw-config.jam
+++ /dev/null
@@ -1,34 +0,0 @@
-#~ Copyright 2005 Rene Rivera.
-#~ Distributed under the Boost Software License, Version 1.0.
-#~ (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-# Automatic configuration for CodeWarrior toolset. To use, just import this module.
-
-import os ;
-import toolset : using ;
-
-if [ os.name ] = NT
-{
- for local R in 9 8 7
- {
- local cw-path = [ W32_GETREG
- "HKEY_LOCAL_MACHINE\\SOFTWARE\\Metrowerks\\CodeWarrior\\Product Versions\\CodeWarrior for Windows R$(R)"
- : "PATH" ] ;
- local cw-version = [ W32_GETREG
- "HKEY_LOCAL_MACHINE\\SOFTWARE\\Metrowerks\\CodeWarrior\\Product Versions\\CodeWarrior for Windows R$(R)"
- : "VERSION" ] ;
- cw-path ?= [ W32_GETREG
- "HKEY_LOCAL_MACHINE\\SOFTWARE\\Metrowerks\\CodeWarrior for Windows\\$(R).0"
- : "PATH" ] ;
- cw-version ?= $(R).0 ;
-
- if $(cw-path)
- {
- if --debug-configuration in [ modules.peek : ARGV ]
- {
- ECHO "notice:" using cw ":" $(cw-version) ":" "$(cw-path)\\Other Metrowerks Tools\\Command Line Tools\\mwcc.exe" ;
- }
- using cw : $(cw-version) : "$(cw-path)\\Other Metrowerks Tools\\Command Line Tools\\mwcc.exe" ;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/cw.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/cw.jam b/ext/kenlm/jam-files/boost-build/tools/cw.jam
deleted file mode 100644
index ddcbfeb..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/cw.jam
+++ /dev/null
@@ -1,246 +0,0 @@
-# Copyright (C) Reece H Dunn 2004
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# based on the msvc.jam toolset
-
-import property ;
-import generators ;
-import os ;
-import type ;
-import toolset : flags ;
-import errors : error ;
-import feature : feature get-values ;
-import path ;
-import sequence : unique ;
-import common ;
-
-if [ MATCH (--debug-configuration) : [ modules.peek : ARGV ] ]
-{
- .debug-configuration = true ;
-}
-
-feature.extend toolset : cw ;
-
-toolset.add-requirements <toolset>cw,<runtime-link>shared:<threading>multi ;
-
-nl = "
-" ;
-
-rule init ( version ? : command * : options * )
-{
- # TODO: fix the $(command[1]) = $(compiler) issue
-
- setup = [ get-values <setup> : $(options) ] ;
- setup ?= cwenv.bat ;
- compiler = [ get-values <compiler> : $(options) ] ;
- compiler ?= mwcc ;
- linker = [ get-values <linker> : $(options) ] ;
- linker ?= mwld ;
-
- local condition = [ common.check-init-parameters cw :
- version $(version) ] ;
-
- command = [ common.get-invocation-command cw : mwcc.exe : $(command) :
- [ default-paths $(version) ] ] ;
-
- common.handle-options cw : $(condition) : $(command) : $(options) ;
-
- local root = [ feature.get-values <root> : $(options) ] ;
- if $(command)
- {
- command = [ common.get-absolute-tool-path $(command[-1]) ] ;
- }
- local tool-root = $(command) ;
-
- setup = $(tool-root)\\$(setup) ;
-
- # map the batch file in setup so it can be executed
-
- other-tools = $(tool-root:D) ;
- root ?= $(other-tools:D) ;
-
- flags cw.link RUN_PATH $(condition) :
- "$(root)\\Win32-x86 Support\\Libraries\\Runtime"
- "$(root)\\Win32-x86 Support\\Libraries\\Runtime\\Libs\\MSL_All-DLLs" ;
-
- setup = "set \"CWFOLDER="$(root)"\" && call \""$(setup)"\" > nul " ;
-
- if [ os.name ] = NT
- {
- setup = $(setup)"
-" ;
- }
- else
- {
- setup = "cmd /S /C "$(setup)" \"&&\" " ;
- }
-
- # bind the setup command to the tool so it can be executed before the
- # command
-
- local prefix = $(setup) ;
-
- flags cw.compile .CC $(condition) : $(prefix)$(compiler) ;
- flags cw.link .LD $(condition) : $(prefix)$(linker) ;
- flags cw.archive .LD $(condition) : $(prefix)$(linker) ;
-
- if [ MATCH ^([89]\\.) : $(version) ]
- {
- if [ os.name ] = NT
- {
- # The runtime libraries
- flags cw.compile CFLAGS <runtime-link>static/<threading>single/<runtime-debugging>off : -runtime ss ;
- flags cw.compile CFLAGS <runtime-link>static/<threading>single/<runtime-debugging>on : -runtime ssd ;
-
- flags cw.compile CFLAGS <runtime-link>static/<threading>multi/<runtime-debugging>off : -runtime sm ;
- flags cw.compile CFLAGS <runtime-link>static/<threading>multi/<runtime-debugging>on : -runtime smd ;
-
- flags cw.compile CFLAGS <runtime-link>shared/<runtime-debugging>off : -runtime dm ;
- flags cw.compile CFLAGS <runtime-link>shared/<runtime-debugging>on : -runtime dmd ;
- }
- }
-}
-
-
-local rule default-paths ( version ? ) # FIXME
-{
- local possible-paths ;
- local ProgramFiles = [ common.get-program-files-dir ] ;
-
- # TODO: add support for cw8 and cw9 detection
-
- local version-6-path = $(ProgramFiles)"\\Metrowerks\\CodeWarrior" ;
- possible-paths += $(version-6-path) ;
-
- # perform post-processing
-
- possible-paths
- = $(possible-paths)"\\Other Metrowerks Tools\\Command Line Tools" ;
-
- possible-paths += [ modules.peek : PATH Path path ] ;
-
- return $(possible-paths) ;
-}
-
-
-
-
-## declare generators
-
-generators.register-c-compiler cw.compile.c++ : CPP : OBJ : <toolset>cw ;
-generators.register-c-compiler cw.compile.c : C : OBJ : <toolset>cw ;
-
-generators.register-linker cw.link
- : OBJ SEARCHED_LIB STATIC_LIB IMPORT_LIB
- : EXE
- : <toolset>cw
- ;
-generators.register-linker cw.link.dll
- : OBJ SEARCHED_LIB STATIC_LIB IMPORT_LIB
- : SHARED_LIB IMPORT_LIB
- : <toolset>cw
- ;
-
-generators.register-archiver cw.archive
- : OBJ
- : STATIC_LIB
- : <toolset>cw
- ;
-
-## compilation phase
-
-flags cw WHATEVER <toolset-cw:version> ;
-
-flags cw.compile CFLAGS <debug-symbols>on : -g ;
-flags cw.compile CFLAGS <optimization>off : -O0 ;
-flags cw.compile CFLAGS <optimization>speed : -O4,p ;
-flags cw.compile CFLAGS <optimization>space : -O4,s ;
-flags cw.compile CFLAGS <inlining>off : -inline off ;
-flags cw.compile CFLAGS <inlining>on : -inline on ;
-flags cw.compile CFLAGS <inlining>full : -inline all ;
-flags cw.compile CFLAGS <exception-handling>off : -Cpp_exceptions off ;
-
-
-flags cw.compile CFLAGS <rtti>on : -RTTI on ;
-flags cw.compile CFLAGS <rtti>off : -RTTI off ;
-
-flags cw.compile CFLAGS <warnings>on : -w on ;
-flags cw.compile CFLAGS <warnings>off : -w off ;
-flags cw.compile CFLAGS <warnings>all : -w all ;
-flags cw.compile CFLAGS <warnings-as-errors>on : -w error ;
-
-flags cw.compile USER_CFLAGS <cflags> : ;
-flags cw.compile.c++ USER_CFLAGS <cxxflags> : ;
-
-flags cw.compile DEFINES <define> ;
-flags cw.compile UNDEFS <undef> ;
-flags cw.compile INCLUDES <include> ;
-
-actions compile.c
-{
- $(.CC) -c -cwd include -lang c -U$(UNDEFS) $(CFLAGS) $(USER_CFLAGS) -I- -o "$(<)" @"@($(<[1]:W).rsp:E=$(nl)"$(>)" $(nl)-D$(DEFINES) $(nl)"-I$(INCLUDES)")"
-}
-actions compile.c++
-{
- $(.CC) -c -cwd include -lang c++ -U$(UNDEFS) $(CFLAGS) $(USER_CFLAGS) -I- -o "$(<)" @"@($(<[1]:W).rsp:E=$(nl)"$(>)" $(nl)-D$(DEFINES) $(nl)"-I$(INCLUDES)")"
-}
-
-## linking phase
-
-flags cw.link DEF_FILE <def-file> ;
-
-flags cw LINKFLAGS : -search ;
-flags cw LINKFLAGS <debug-symbols>on : -g ;
-flags cw LINKFLAGS <user-interface>console : -subsystem console ;
-flags cw LINKFLAGS <user-interface>gui : -subsystem windows ;
-flags cw LINKFLAGS <user-interface>wince : -subsystem wince ;
-flags cw LINKFLAGS <user-interface>native : -subsystem native ;
-flags cw LINKFLAGS <user-interface>auto : -subsystem auto ;
-
-flags cw LINKFLAGS <main-target-type>LIB/<link>static : -library ;
-
-flags cw.link USER_LINKFLAGS <linkflags> ;
-flags cw.link LINKPATH <library-path> ;
-
-flags cw.link FINDLIBS_ST <find-static-library> ;
-flags cw.link FINDLIBS_SA <find-shared-library> ;
-flags cw.link LIBRARY_OPTION <toolset>cw : "" : unchecked ;
-flags cw.link LIBRARIES_MENTIONED_BY_FILE : <library-file> ;
-
-rule link.dll ( targets + : sources * : properties * )
-{
- DEPENDS $(<) : [ on $(<) return $(DEF_FILE) ] ;
-}
-
-if [ os.name ] in NT
-{
- actions archive
- {
- if exist "$(<[1])" DEL "$(<[1])"
- $(.LD) -library -o "$(<[1])" @"@($(<[1]:W).rsp:E=$(nl)"$(>)" $(nl)$(LIBRARIES_MENTIONED_BY_FILE) $(nl)"$(LIBRARY_OPTION)$(FINDLIBS_ST:S=.lib)" $(nl)"$(LIBRARY_OPTION)$(FINDLIBS_SA:S=.lib)")"
- }
-}
-else # cygwin
-{
- actions archive
- {
- _bbv2_out_="$(<)"
- if test -f "$_bbv2_out_" ; then
- _bbv2_existing_="$(<:W)"
- fi
- $(.LD) -library -o "$(<:W)" $_bbv2_existing_ @"@($(<[1]:W).rsp:E=$(nl)"$(>)" $(nl)$(LIBRARIES_MENTIONED_BY_FILE) $(nl)"$(LIBRARY_OPTION)$(FINDLIBS_ST:S=.lib)" $(nl)"$(LIBRARY_OPTION)$(FINDLIBS_SA:S=.lib)")"
- }
-}
-
-actions link bind DEF_FILE
-{
- $(.LD) -o "$(<[1]:W)" -L"$(LINKPATH)" $(LINKFLAGS) $(USER_LINKFLAGS) @"@($(<[1]:W).rsp:E=$(nl)"$(>)" $(nl)$(LIBRARIES_MENTIONED_BY_FILE) $(nl)"$(LIBRARY_OPTION)$(FINDLIBS_ST:S=.lib)" $(nl)"$(LIBRARY_OPTION)$(FINDLIBS_SA:S=.lib)")"
-}
-
-actions link.dll bind DEF_FILE
-{
- $(.LD) -shared -o "$(<[1]:W)" -implib "$(<[2]:W)" -L"$(LINKPATH)" $(LINKFLAGS) -f"$(DEF_FILE)" $(USER_LINKFLAGS) @"@($(<[1]:W).rsp:E=$(nl)"$(>)" $(nl)$(LIBRARIES_MENTIONED_BY_FILE) $(nl)"$(LIBRARY_OPTION)$(FINDLIBS_ST:S=.lib)" $(nl)"$(LIBRARY_OPTION)$(FINDLIBS_SA:S=.lib)")"
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/darwin.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/darwin.jam b/ext/kenlm/jam-files/boost-build/tools/darwin.jam
deleted file mode 100644
index bd77ff7..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/darwin.jam
+++ /dev/null
@@ -1,592 +0,0 @@
-# Copyright 2003 Christopher Currie
-# Copyright 2006 Dave Abrahams
-# Copyright 2003, 2004, 2005, 2006 Vladimir Prus
-# Copyright 2005-2007 Mat Marcus
-# Copyright 2005-2007 Adobe Systems Incorporated
-# Copyright 2007-2010 Rene Rivera
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-# Please see http://article.gmane.org/gmane.comp.lib.boost.build/3389/
-# for explanation why it's a separate toolset.
-
-import feature : feature ;
-import toolset : flags ;
-import type ;
-import common ;
-import generators ;
-import path : basename ;
-import version ;
-import property-set ;
-import regex ;
-import errors ;
-
-## Use a framework.
-feature framework : : free ;
-
-## The MacOSX version to compile for, which maps to the SDK to use (sysroot).
-feature macosx-version : : propagated link-incompatible symmetric optional ;
-
-## The minimal MacOSX version to target.
-feature macosx-version-min : : propagated optional ;
-
-## A dependency, that is forced to be included in the link.
-feature force-load : : free dependency incidental ;
-
-#############################################################################
-
-_ = " " ;
-
-if [ MATCH (--debug-configuration) : [ modules.peek : ARGV ] ]
-{
- .debug-configuration = true ;
-}
-
-feature.extend toolset : darwin ;
-import gcc ;
-toolset.inherit-generators darwin : gcc : gcc.mingw.link gcc.mingw.link.dll ;
-
-generators.override darwin.prebuilt : builtin.prebuilt ;
-generators.override darwin.searched-lib-generator : searched-lib-generator ;
-
-# Override default do-nothing generators.
-generators.override darwin.compile.c.pch : pch.default-c-pch-generator ;
-generators.override darwin.compile.c++.pch : pch.default-cpp-pch-generator ;
-
-type.set-generated-target-suffix PCH : <toolset>darwin : gch ;
-
-toolset.inherit-rules darwin : gcc : localize ;
-toolset.inherit-flags darwin : gcc
- : <runtime-link>static
- <architecture>arm/<address-model>32
- <architecture>arm/<address-model>64
- <architecture>arm/<instruction-set>
- <architecture>x86/<address-model>32
- <architecture>x86/<address-model>64
- <architecture>x86/<instruction-set>
- <architecture>power/<address-model>32
- <architecture>power/<address-model>64
- <architecture>power/<instruction-set> ;
-
-# Options:
-#
-# <root>PATH
-# Platform root path. The common autodetection will set this to
-# "/Developer". And when a command is given it will be set to
-# the corresponding "*.platform/Developer" directory.
-#
-rule init ( version ? : command * : options * : requirement * )
-{
- # First time around, figure what is host OSX version
- if ! $(.host-osx-version)
- {
- .host-osx-version = [ MATCH "^([0-9.]+)"
- : [ SHELL "/usr/bin/sw_vers -productVersion" ] ] ;
- if $(.debug-configuration)
- {
- ECHO notice: OSX version on this machine is $(.host-osx-version) ;
- }
- }
-
- # - The root directory of the tool install.
- local root = [ feature.get-values <root> : $(options) ] ;
-
- # - The bin directory where to find the commands to execute.
- local bin ;
-
- # - The configured compile driver command.
- local command = [ common.get-invocation-command darwin : g++ : $(command) ] ;
-
- # The version as reported by the compiler
- local real-version ;
-
- # - Autodetect the root and bin dir if not given.
- if $(command)
- {
- bin ?= [ common.get-absolute-tool-path $(command[1]) ] ;
- if $(bin) = "/usr/bin"
- {
- root ?= /Developer ;
- }
- else
- {
- local r = $(bin:D) ;
- r = $(r:D) ;
- root ?= $(r) ;
- }
- }
-
- # - Autodetect the version if not given.
- if $(command)
- {
- # - The 'command' variable can have multiple elements. When calling
- # the SHELL builtin we need a single string.
- local command-string = $(command:J=" ") ;
- real-version = [ MATCH "^([0-9.]+)"
- : [ SHELL "$(command-string) -dumpversion" ] ] ;
- version ?= $(real-version) ;
- }
-
- .real-version.$(version) = $(real-version) ;
-
- # - Define the condition for this toolset instance.
- local condition =
- [ common.check-init-parameters darwin $(requirement) : version $(version) ] ;
-
- # - Set the toolset generic common options.
- common.handle-options darwin : $(condition) : $(command) : $(options) ;
-
- # - GCC 4.0 and higher in Darwin does not have -fcoalesce-templates.
- if $(real-version) < "4.0.0"
- {
- flags darwin.compile.c++ OPTIONS $(condition) : -fcoalesce-templates ;
- }
- # - GCC 4.2 and higher in Darwin does not have -Wno-long-double.
- if $(real-version) < "4.2.0"
- {
- flags darwin.compile OPTIONS $(condition) : -Wno-long-double ;
- }
-
- # - Set the link flags common with the GCC toolset.
- gcc.init-link-flags darwin darwin $(condition) ;
-
- # - The symbol strip program.
- local strip ;
- if <striper> in $(options)
- {
- # We can turn off strip by specifying it as empty. In which
- # case we switch to using the linker to do the strip.
- flags darwin.link.dll OPTIONS
- $(condition)/<main-target-type>LIB/<link>shared/<address-model>32/<strip>on : -Wl,-x ;
- flags darwin.link.dll OPTIONS
- $(condition)/<main-target-type>LIB/<link>shared/<address-model>/<strip>on : -Wl,-x ;
- flags darwin.link OPTIONS
- $(condition)/<main-target-type>EXE/<address-model>32/<strip>on : -s ;
- flags darwin.link OPTIONS
- $(condition)/<main-target-type>EXE/<address-model>/<strip>on : -s ;
- }
- else
- {
- # Otherwise we need to find a strip program to use. And hence
- # also tell the link action that we need to use a strip
- # post-process.
- flags darwin.link NEED_STRIP $(condition)/<strip>on : "" ;
- strip =
- [ common.get-invocation-command darwin
- : strip : [ feature.get-values <striper> : $(options) ] : $(bin) : search-path ] ;
- flags darwin.link .STRIP $(condition) : $(strip[1]) ;
- if $(.debug-configuration)
- {
- ECHO notice: using strip for $(condition) at $(strip[1]) ;
- }
- }
-
- # - The archive builder (libtool is the default as creating
- # archives in darwin is complicated.
- local archiver =
- [ common.get-invocation-command darwin
- : libtool : [ feature.get-values <archiver> : $(options) ] : $(bin) : search-path ] ;
- flags darwin.archive .LIBTOOL $(condition) : $(archiver[1]) ;
- if $(.debug-configuration)
- {
- ECHO notice: using archiver for $(condition) at $(archiver[1]) ;
- }
-
- # - Initialize the SDKs available in the root for this tool.
- local sdks = [ init-available-sdk-versions $(condition) : $(root) ] ;
-
- #~ ECHO --- ;
- #~ ECHO --- bin :: $(bin) ;
- #~ ECHO --- root :: $(root) ;
- #~ ECHO --- version :: $(version) ;
- #~ ECHO --- condition :: $(condition) ;
- #~ ECHO --- strip :: $(strip) ;
- #~ ECHO --- archiver :: $(archiver) ;
- #~ ECHO --- sdks :: $(sdks) ;
- #~ ECHO --- ;
- #~ EXIT ;
-}
-
-# Add and set options for a discovered SDK version.
-local rule init-sdk ( condition * : root ? : version + : version-feature ? )
-{
- local rule version-to-feature ( version + )
- {
- switch $(version[1])
- {
- case iphone* :
- {
- return $(version[1])-$(version[2-]:J=.) ;
- }
- case mac* :
- {
- return $(version[2-]:J=.) ;
- }
- case * :
- {
- return $(version:J=.) ;
- }
- }
- }
-
- # leave compiler flags etc. up to the toolchain
- return $(version-feature) ;
-
- if $(version-feature)
- {
- if $(.debug-configuration)
- {
- ECHO notice: available sdk for $(condition)/<macosx-version>$(version-feature) at $(root) ;
- }
-
- # Add the version to the features for specifying them.
- if ! $(version-feature) in [ feature.values macosx-version ]
- {
- feature.extend macosx-version : $(version-feature) ;
- }
- if ! $(version-feature) in [ feature.values macosx-version-min ]
- {
- feature.extend macosx-version-min : $(version-feature) ;
- }
-
- # Set the flags the version needs to compile with, first
- # generic options.
- flags darwin.compile OPTIONS $(condition)/<macosx-version>$(version-feature)
- : -isysroot $(root) ;
- flags darwin.link OPTIONS $(condition)/<macosx-version>$(version-feature)
- : -isysroot $(root) ;
-
- # Then device variation options.
- switch $(version[1])
- {
- case iphonesim* :
- {
- local N = $(version[2]) ;
- if ! $(version[3]) { N += 00 ; }
- else if [ regex.match (..) : $(version[3]) ] { N += $(version[3]) ; }
- else { N += 0$(version[3]) ; }
- if ! $(version[4]) { N += 00 ; }
- else if [ regex.match (..) : $(version[4]) ] { N += $(version[4]) ; }
- else { N += 0$(version[4]) ; }
- N = $(N:J=) ;
- flags darwin.compile OPTIONS <macosx-version-min>$(version-feature)
- : -D__IPHONE_OS_VERSION_MIN_REQUIRED=$(N) ;
- flags darwin.link OPTIONS <macosx-version-min>$(version-feature)
- : -D__IPHONE_OS_VERSION_MIN_REQUIRED=$(N) ;
- }
-
- case iphone* :
- {
- flags darwin.compile OPTIONS <macosx-version-min>$(version-feature)
- : -miphoneos-version-min=$(version[2-]:J=.) ;
- flags darwin.link OPTIONS <macosx-version-min>$(version-feature)
- : -miphoneos-version-min=$(version[2-]:J=.) ;
- }
-
- case mac* :
- {
- flags darwin.compile OPTIONS <macosx-version-min>$(version-feature)
- : -mmacosx-version-min=$(version[2-]:J=.) ;
- flags darwin.link OPTIONS <macosx-version-min>$(version-feature)
- : -mmacosx-version-min=$(version[2-]:J=.) ;
- }
- }
-
- if $(version[3]) > 0
- {
- # We have a minor version of an SDK. We want to set up
- # previous minor versions, plus the current minor version.
- # So we recurse to set up the previous minor versions, up to
- # the current version.
- local minor-minus-1 = [ CALC $(version[3]) - 1 ] ;
- return
- [ init-sdk $(condition) : $(root)
- : $(version[1-2]) $(minor-minus-1) : [ version-to-feature $(version[1-2]) $(minor-minus-1) ] ]
- $(version-feature) ;
- }
- else
- {
- return $(version-feature) ;
- }
- }
- else if $(version[4])
- {
- # We have a patch version of an SDK. We want to set up
- # both the specific patch version, and the minor version.
- # So we recurse to set up the patch version. Plus the minor version.
- return
- [ init-sdk $(condition) : $(root)
- : $(version[1-3]) : [ version-to-feature $(version[1-3]) ] ]
- [ init-sdk $(condition) : $(root)
- : $(version) : [ version-to-feature $(version) ] ] ;
- }
- else
- {
- # Yes, this is intentionally recursive.
- return
- [ init-sdk $(condition) : $(root)
- : $(version) : [ version-to-feature $(version) ] ] ;
- }
-}
-
-# Determine the MacOSX SDK versions installed and their locations.
-local rule init-available-sdk-versions ( condition * : root ? )
-{
- root ?= /Developer ;
- local sdks-root = $(root)/SDKs ;
- local sdks = [ GLOB $(sdks-root) : MacOSX*.sdk iPhoneOS*.sdk iPhoneSimulator*.sdk ] ;
- local result ;
- for local sdk in $(sdks)
- {
- local sdk-match = [ MATCH ([^0-9]+)([0-9]+)[.]([0-9x]+)[.]?([0-9x]+)? : $(sdk:D=) ] ;
- local sdk-platform = $(sdk-match[1]:L) ;
- local sdk-version = $(sdk-match[2-]) ;
- if $(sdk-version)
- {
- switch $(sdk-platform)
- {
- case macosx :
- {
- sdk-version = mac $(sdk-version) ;
- }
- case iphoneos :
- {
- sdk-version = iphone $(sdk-version) ;
- }
- case iphonesimulator :
- {
- sdk-version = iphonesim $(sdk-version) ;
- }
- case * :
- {
- sdk-version = $(sdk-version:J=-) ;
- }
- }
- result += [ init-sdk $(condition) : $(sdk) : $(sdk-version) ] ;
- }
- }
- return $(result) ;
-}
-
-# Generic options.
-flags darwin.compile OPTIONS <flags> ;
-
-# The following adds objective-c support to darwin.
-# Thanks to http://thread.gmane.org/gmane.comp.lib.boost.build/13759
-
-generators.register-c-compiler darwin.compile.m : OBJECTIVE_C : OBJ : <toolset>darwin ;
-generators.register-c-compiler darwin.compile.mm : OBJECTIVE_CPP : OBJ : <toolset>darwin ;
-
-rule setup-address-model ( targets * : sources * : properties * )
-{
- local ps = [ property-set.create $(properties) ] ;
- local arch = [ $(ps).get <architecture> ] ;
- local instruction-set = [ $(ps).get <instruction-set> ] ;
- local address-model = [ $(ps).get <address-model> ] ;
- local osx-version = [ $(ps).get <macosx-version> ] ;
- local gcc-version = [ $(ps).get <toolset-darwin:version> ] ;
- gcc-version = $(.real-version.$(gcc-version)) ;
- local options ;
-
- local support-ppc64 = 1 ;
-
- osx-version ?= $(.host-osx-version) ;
-
- switch $(osx-version)
- {
- case iphone* :
- {
- support-ppc64 = ;
- }
-
- case * :
- if $(osx-version) && ! [ version.version-less [ regex.split $(osx-version) \\. ] : 10 6 ]
- {
- # When targeting 10.6:
- # - gcc 4.2 will give a compiler errir if ppc64 compilation is requested
- # - gcc 4.0 will compile fine, somehow, but then fail at link time
- support-ppc64 = ;
- }
- }
- # Gentoo Prefix toolchain doesn't do multi-arch, so don't try either
- switch $(donotaddarchpleaseXXXarch)
- {
- case combined :
- {
- if $(address-model) = 32_64 {
- if $(support-ppc64) {
- options = -arch i386 -arch ppc -arch x86_64 -arch ppc64 ;
- } else {
- # Build 3-way binary
- options = -arch i386 -arch ppc -arch x86_64 ;
- }
- } else if $(address-model) = 64 {
- if $(support-ppc64) {
- options = -arch x86_64 -arch ppc64 ;
- } else {
- errors.user-error "64-bit PPC compilation is not supported when targeting OSX 10.6 or later" ;
- }
- } else {
- options = -arch i386 -arch ppc ;
- }
- }
-
- case x86 :
- {
- if $(address-model) = 32_64 {
- options = -arch i386 -arch x86_64 ;
- } else if $(address-model) = 64 {
- options = -arch x86_64 ;
- } else {
- options = -arch i386 ;
- }
- }
-
- case power :
- {
- if ! $(support-ppc64)
- && ( $(address-model) = 32_64 || $(address-model) = 64 )
- {
- errors.user-error "64-bit PPC compilation is not supported when targeting OSX 10.6 or later" ;
- }
-
- if $(address-model) = 32_64 {
- options = -arch ppc -arch ppc64 ;
- } else if $(address-model) = 64 {
- options = -arch ppc64 ;
- } else {
- options = -arch ppc ;
- }
- }
-
- case arm :
- {
- if $(instruction-set) {
- options = -arch$(_)$(instruction-set) ;
- } else {
- options = -arch arm ;
- }
- }
- }
-
- if $(options)
- {
- OPTIONS on $(targets) += $(options) ;
- }
-}
-
-rule setup-threading ( targets * : sources * : properties * )
-{
- gcc.setup-threading $(targets) : $(sources) : $(properties) ;
-}
-
-rule setup-fpic ( targets * : sources * : properties * )
-{
- gcc.setup-fpic $(targets) : $(sources) : $(properties) ;
-}
-
-rule compile.m ( targets * : sources * : properties * )
-{
- LANG on $(<) = "-x objective-c" ;
- gcc.setup-fpic $(targets) : $(sources) : $(properties) ;
- setup-address-model $(targets) : $(sources) : $(properties) ;
-}
-
-actions compile.m
-{
- "$(CONFIG_COMMAND)" $(LANG) $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
-}
-
-rule compile.mm ( targets * : sources * : properties * )
-{
- LANG on $(<) = "-x objective-c++" ;
- gcc.setup-fpic $(targets) : $(sources) : $(properties) ;
- setup-address-model $(targets) : $(sources) : $(properties) ;
-}
-
-actions compile.mm
-{
- "$(CONFIG_COMMAND)" $(LANG) $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
-}
-
-# Set the max header padding to allow renaming of libs for installation.
-flags darwin.link.dll OPTIONS : -headerpad_max_install_names ;
-
-# To link the static runtime we need to link to all the core runtime libraries.
-flags darwin.link OPTIONS <runtime-link>static
- : -nodefaultlibs -shared-libgcc -lstdc++-static -lgcc_eh -lgcc -lSystem ;
-
-# Strip as much as possible when optimizing.
-flags darwin.link OPTIONS <optimization>speed : -Wl,-dead_strip -no_dead_strip_inits_and_terms ;
-flags darwin.link OPTIONS <optimization>space : -Wl,-dead_strip -no_dead_strip_inits_and_terms ;
-
-# Dynamic/shared linking.
-flags darwin.compile OPTIONS <link>shared : -dynamic ;
-
-# Misc options.
-flags darwin.compile OPTIONS : -gdwarf-2 -fexceptions ;
-#~ flags darwin.link OPTIONS : -fexceptions ;
-
-# Add the framework names to use.
-flags darwin.link FRAMEWORK <framework> ;
-
-#
-flags darwin.link FORCE_LOAD <force-load> ;
-
-# This is flag is useful for debugging the link step
-# uncomment to see what libtool is doing under the hood
-#~ flags darwin.link.dll OPTIONS : -Wl,-v ;
-
-# set up the -F option to include the paths to any frameworks used.
-local rule prepare-framework-path ( target + )
-{
- # The -framework option only takes basename of the framework.
- # The -F option specifies the directories where a framework
- # is searched for. So, if we find <framework> feature
- # with some path, we need to generate property -F option.
- local framework-paths = [ on $(target) return $(FRAMEWORK:D) ] ;
-
- # Be sure to generate no -F if there's no path.
- for local framework-path in $(framework-paths)
- {
- if $(framework-path) != ""
- {
- FRAMEWORK_PATH on $(target) += -F$(framework-path) ;
- }
- }
-}
-
-rule link ( targets * : sources * : properties * )
-{
- DEPENDS $(targets) : [ on $(targets) return $(FORCE_LOAD) ] ;
- setup-address-model $(targets) : $(sources) : $(properties) ;
- prepare-framework-path $(<) ;
-}
-
-# Note that using strip without any options was reported to result in broken
-# binaries, at least on OS X 10.5.5, see:
-# http://svn.boost.org/trac/boost/ticket/2347
-# So we pass -S -x.
-actions link bind LIBRARIES FORCE_LOAD
-{
- "$(CONFIG_COMMAND)" -L"$(LINKPATH)" -o "$(<)" "$(>)" -Wl,-force_load$(_)"$(FORCE_LOAD)" "$(LIBRARIES)" -l$(FINDLIBS-SA) -l$(FINDLIBS-ST) $(FRAMEWORK_PATH) -framework$(_)$(FRAMEWORK:D=:S=) $(OPTIONS) $(USER_OPTIONS)
- $(NEED_STRIP)"$(.STRIP)" $(NEED_STRIP)-S $(NEED_STRIP)-x $(NEED_STRIP)"$(<)"
-}
-
-rule link.dll ( targets * : sources * : properties * )
-{
- setup-address-model $(targets) : $(sources) : $(properties) ;
- prepare-framework-path $(<) ;
-}
-
-actions link.dll bind LIBRARIES
-{
- "$(CONFIG_COMMAND)" -dynamiclib -Wl,-single_module -install_name "$(<:B)$(<:S)" -L"$(LINKPATH)" -o "$(<)" "$(>)" "$(LIBRARIES)" -l$(FINDLIBS-SA) -l$(FINDLIBS-ST) $(FRAMEWORK_PATH) -framework$(_)$(FRAMEWORK:D=:S=) $(OPTIONS) $(USER_OPTIONS)
-}
-
-# We use libtool instead of ar to support universal binary linking
-# TODO: Find a way to use the underlying tools, i.e. lipo, to do this.
-actions piecemeal archive
-{
- "$(.LIBTOOL)" -static -o "$(<:T)" $(ARFLAGS) "$(>:T)"
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/dmc.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/dmc.jam b/ext/kenlm/jam-files/boost-build/tools/dmc.jam
deleted file mode 100644
index 8af8725..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/dmc.jam
+++ /dev/null
@@ -1,134 +0,0 @@
-# Digital Mars C++
-
-# (C) Copyright Christof Meerwald 2003.
-# (C) Copyright Aleksey Gurtovoy 2004.
-# (C) Copyright Arjan Knepper 2006.
-#
-# Distributed under the Boost Software License, Version 1.0. (See
-# accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# The following #// line will be used by the regression test table generation
-# program as the column heading for HTML tables. Must not include version number.
-#//<a href="http://www.digitalmars.com/">Digital<br>Mars C++</a>
-
-import feature generators common ;
-import toolset : flags ;
-import sequence regex ;
-
-feature.extend toolset : dmc ;
-
-rule init ( version ? : command * : options * )
-{
- local condition = [ common.check-init-parameters dmc : version $(version) ] ;
-
- local command = [ common.get-invocation-command dmc : dmc : $(command) ] ;
- command ?= dmc ;
-
- common.handle-options dmc : $(condition) : $(command) : $(options) ;
-
- if $(command)
- {
- command = [ common.get-absolute-tool-path $(command[-1]) ] ;
- }
- root = $(command:D) ;
-
- if $(root)
- {
- # DMC linker is sensitive the the direction of slashes, and
- # won't link if forward slashes are used in command.
- root = [ sequence.join [ regex.split $(root) "/" ] : "\\" ] ;
- flags dmc .root $(condition) : $(root)\\bin\\ ;
- }
- else
- {
- flags dmc .root $(condition) : "" ;
- }
-}
-
-
-# Declare generators
-generators.register-linker dmc.link : OBJ SEARCHED_LIB STATIC_LIB IMPORT_LIB : EXE : <toolset>dmc ;
-generators.register-linker dmc.link.dll : OBJ SEARCHED_LIB STATIC_LIB IMPORT_LIB : SHARED_LIB IMPORT_LIB : <toolset>dmc ;
-
-generators.register-archiver dmc.archive : OBJ : STATIC_LIB : <toolset>dmc ;
-generators.register-c-compiler dmc.compile.c++ : CPP : OBJ : <toolset>dmc ;
-generators.register-c-compiler dmc.compile.c : C : OBJ : <toolset>dmc ;
-
-
-# Declare flags
-# dmc optlink has some limitation on the amount of debug-info included. Therefore only linenumbers are enabled in debug builds.
-# flags dmc.compile OPTIONS <debug-symbols>on : -g ;
-flags dmc.compile OPTIONS <debug-symbols>on : -gl ;
-flags dmc.link OPTIONS <debug-symbols>on : /CO /NOPACKF /DEBUGLI ;
-flags dmc.link OPTIONS <debug-symbols>off : /PACKF ;
-
-flags dmc.compile OPTIONS <optimization>off : -S -o+none ;
-flags dmc.compile OPTIONS <optimization>speed : -o+time ;
-flags dmc.compile OPTIONS <optimization>space : -o+space ;
-flags dmc.compile OPTIONS <exception-handling>on : -Ae ;
-flags dmc.compile OPTIONS <rtti>on : -Ar ;
-# FIXME:
-# Compiling sources to be linked into a shared lib (dll) the -WD cflag should be used
-# Compiling sources to be linked into a static lib (lib) or executable the -WA cflag should be used
-# But for some reason the -WD cflag is always in use.
-# flags dmc.compile OPTIONS <link>shared : -WD ;
-# flags dmc.compile OPTIONS <link>static : -WA ;
-
-# Note that these two options actually imply multithreading support on DMC
-# because there is no single-threaded dynamic runtime library. Specifying
-# <threading>multi would be a bad idea, though, because no option would be
-# matched when the build uses the default settings of <runtime-link>dynamic
-# and <threading>single.
-flags dmc.compile OPTIONS <runtime-debugging>off/<runtime-link>shared : -ND ;
-flags dmc.compile OPTIONS <runtime-debugging>on/<runtime-link>shared : -ND ;
-
-flags dmc.compile OPTIONS <runtime-debugging>off/<runtime-link>static/<threading>single : ;
-flags dmc.compile OPTIONS <runtime-debugging>on/<runtime-link>static/<threading>single : ;
-flags dmc.compile OPTIONS <runtime-debugging>off/<runtime-link>static/<threading>multi : -D_MT ;
-flags dmc.compile OPTIONS <runtime-debugging>on/<runtime-link>static/<threading>multi : -D_MT ;
-
-flags dmc.compile OPTIONS : <cflags> ;
-flags dmc.compile.c++ OPTIONS : <cxxflags> ;
-
-flags dmc.compile DEFINES : <define> ;
-flags dmc.compile INCLUDES : <include> ;
-
-flags dmc.link <linkflags> ;
-flags dmc.archive OPTIONS <arflags> ;
-
-flags dmc LIBPATH <library-path> ;
-flags dmc LIBRARIES <library-file> ;
-flags dmc FINDLIBS <find-library-sa> ;
-flags dmc FINDLIBS <find-library-st> ;
-
-actions together link bind LIBRARIES
-{
- "$(.root)link" $(OPTIONS) /NOI /DE /XN "$(>)" , "$(<[1])" ,, $(LIBRARIES) user32.lib kernel32.lib "$(FINDLIBS:S=.lib)" , "$(<[2]:B).def"
-}
-
-actions together link.dll bind LIBRARIES
-{
- echo LIBRARY "$(<[1])" > $(<[2]:B).def
- echo DESCRIPTION 'A Library' >> $(<[2]:B).def
- echo EXETYPE NT >> $(<[2]:B).def
- echo SUBSYSTEM WINDOWS >> $(<[2]:B).def
- echo CODE EXECUTE READ >> $(<[2]:B).def
- echo DATA READ WRITE >> $(<[2]:B).def
- "$(.root)link" $(OPTIONS) /NOI /DE /XN /ENTRY:_DllMainCRTStartup /IMPLIB:"$(<[2])" "$(>)" $(LIBRARIES) , "$(<[1])" ,, user32.lib kernel32.lib "$(FINDLIBS:S=.lib)" , "$(<[2]:B).def"
-}
-
-actions compile.c
-{
- "$(.root)dmc" -c $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -o"$(<)" "$(>)"
-}
-
-actions compile.c++
-{
- "$(.root)dmc" -cpp -c -Ab $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -o"$(<)" "$(>)"
-}
-
-actions together piecemeal archive
-{
- "$(.root)lib" $(OPTIONS) -c -n -p256 "$(<)" "$(>)"
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/docutils.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/docutils.jam b/ext/kenlm/jam-files/boost-build/tools/docutils.jam
deleted file mode 100644
index fc775b6..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/docutils.jam
+++ /dev/null
@@ -1,85 +0,0 @@
-# Copyright David Abrahams 2004. Distributed under the Boost
-# Software License, Version 1.0. (See accompanying
-# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
-
-# Support for docutils ReStructuredText processing.
-
-import type ;
-import scanner ;
-import generators ;
-import os ;
-import common ;
-import toolset ;
-import path ;
-import feature : feature ;
-import property ;
-
-.initialized = ;
-
-type.register ReST : rst ;
-
-class rst-scanner : common-scanner
-{
- rule __init__ ( paths * )
- {
- common-scanner.__init__ . $(paths) ;
- }
-
- rule pattern ( )
- {
- return "^[ ]*\\.\\.[ ]+include::[ ]+([^
-]+)"
- "^[ ]*\\.\\.[ ]+image::[ ]+([^
-]+)"
- "^[ ]*\\.\\.[ ]+figure::[ ]+([^
-]+)"
- ;
- }
-}
-
-scanner.register rst-scanner : include ;
-type.set-scanner ReST : rst-scanner ;
-
-generators.register-standard docutils.html : ReST : HTML ;
-
-rule init ( docutils-dir ? : tools-dir ? )
-{
- docutils-dir ?= [ modules.peek : DOCUTILS_DIR ] ;
- tools-dir ?= $(docutils-dir)/tools ;
-
- if ! $(.initialized)
- {
- .initialized = true ;
- .docutils-dir = $(docutils-dir) ;
- .tools-dir = $(tools-dir:R="") ;
-
- .setup = [
- common.prepend-path-variable-command PYTHONPATH
- : $(.docutils-dir) $(.docutils-dir)/extras ] ;
- RST2XXX = [ common.find-tool rst2html ] ;
- }
-}
-
-rule html ( target : source : properties * )
-{
- if ! [ on $(target) return $(RST2XXX) ]
- {
- local python-cmd = [ property.select <python.interpreter> : $(properties) ] ;
- RST2XXX on $(target) = $(python-cmd:G=:E="python") $(.tools-dir)/rst2html.py ;
- }
-}
-
-
-feature docutils : : free ;
-feature docutils-html : : free ;
-feature docutils-cmd : : free ;
-toolset.flags docutils COMMON-FLAGS : <docutils> ;
-toolset.flags docutils HTML-FLAGS : <docutils-html> ;
-toolset.flags docutils RST2XXX : <docutils-cmd> ;
-
-actions html
-{
- $(.setup)
- "$(RST2XXX)" $(COMMON-FLAGS) $(HTML-FLAGS) $(>) $(<)
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/doxygen-config.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/doxygen-config.jam b/ext/kenlm/jam-files/boost-build/tools/doxygen-config.jam
deleted file mode 100644
index 2cd2cca..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/doxygen-config.jam
+++ /dev/null
@@ -1,11 +0,0 @@
-#~ Copyright 2005, 2006 Rene Rivera.
-#~ Distributed under the Boost Software License, Version 1.0.
-#~ (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-# Automatic configuration for Doxygen tools. To use, just import this module.
-
-import toolset : using ;
-
-ECHO "warning: doxygen-config.jam is deprecated. Use 'using doxygen ;' instead." ;
-
-using doxygen ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/doxygen.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/doxygen.jam b/ext/kenlm/jam-files/boost-build/tools/doxygen.jam
deleted file mode 100644
index 6a56ccd..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/doxygen.jam
+++ /dev/null
@@ -1,775 +0,0 @@
-# Copyright 2003, 2004 Douglas Gregor
-# Copyright 2003, 2004, 2005 Vladimir Prus
-# Copyright 2006 Rene Rivera
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# This module defines rules to handle generation of various outputs from source
-# files documented with doxygen comments. The supported transformations are:
-#
-# * Source -> Doxygen XML -> BoostBook XML
-# * Source -> Doxygen HTML
-#
-# The type of transformation is selected based on the target requested. For
-# BoostBook XML, the default, specifying a target with an ".xml" suffix, or an
-# empty suffix, will produce a <target>.xml and <target>.boostbook. For Doxygen
-# HTML specifying a target with an ".html" suffix will produce a directory
-# <target> with the Doxygen html files, and a <target>.html file redirecting to
-# that directory.
-
-import alias ;
-import boostbook ;
-import "class" : new ;
-import common ;
-import feature ;
-import make ;
-import modules ;
-import generators ;
-import os ;
-import path ;
-import print ;
-import project ;
-import property ;
-import stage ;
-import targets ;
-import toolset ;
-import type ;
-import utility ;
-import xsltproc ;
-import virtual-target ;
-
-
-# Use to specify extra configuration paramters. These get translated into a
-# doxyfile which configures the building of the docs.
-feature.feature doxygen:param : : free ;
-
-# Specify the "<xsl:param>boost.doxygen.header.prefix" XSLT option.
-feature.feature prefix : : free ;
-
-# Specify the "<xsl:param>boost.doxygen.reftitle" XSLT option.
-feature.feature reftitle : : free ;
-
-# Which processor to use for various translations from Doxygen.
-feature.feature doxygen.processor : xsltproc doxproc : propagated implicit ;
-
-# To generate, or not, index sections.
-feature.feature doxygen.doxproc.index : no yes : propagated incidental ;
-
-# The ID for the resulting BoostBook reference section.
-feature.feature doxygen.doxproc.id : : free ;
-
-# The title for the resulting BoostBook reference section.
-feature.feature doxygen.doxproc.title : : free ;
-
-# Location for images when generating XML
-feature.feature doxygen:xml-imagedir : : free ;
-
-# Indicates whether the entire directory should be deleted
-feature.feature doxygen.rmdir : off on : optional incidental ;
-
-# Doxygen configuration input file.
-type.register DOXYFILE : doxyfile ;
-
-# Doxygen XML multi-file output.
-type.register DOXYGEN_XML_MULTIFILE : xml-dir : XML ;
-
-# Doxygen XML coallesed output.
-type.register DOXYGEN_XML : doxygen : XML ;
-
-# Doxygen HTML multifile directory.
-type.register DOXYGEN_HTML_MULTIFILE : html-dir : HTML ;
-
-# Redirection HTML file to HTML multifile directory.
-type.register DOXYGEN_HTML : : HTML ;
-
-type.register DOXYGEN_XML_IMAGES : doxygen-xml-images ;
-
-
-# Initialize the Doxygen module. Parameters are:
-# name: the name of the 'doxygen' executable. If not specified, the name
-# 'doxygen' will be used
-#
-rule init ( name ? )
-{
- if ! $(.initialized)
- {
- .initialized = true ;
-
- .doxproc = [ modules.binding $(__name__) ] ;
- .doxproc = $(.doxproc:D)/doxproc.py ;
-
- generators.register-composing doxygen.headers-to-doxyfile
- : H HPP CPP : DOXYFILE ;
- generators.register-standard doxygen.run
- : DOXYFILE : DOXYGEN_XML_MULTIFILE ;
- generators.register-standard doxygen.xml-dir-to-boostbook
- : DOXYGEN_XML_MULTIFILE : BOOSTBOOK : <doxygen.processor>doxproc ;
- generators.register-standard doxygen.xml-to-boostbook
- : DOXYGEN_XML : BOOSTBOOK : <doxygen.processor>xsltproc ;
- generators.register-standard doxygen.collect
- : DOXYGEN_XML_MULTIFILE : DOXYGEN_XML ;
- generators.register-standard doxygen.run
- : DOXYFILE : DOXYGEN_HTML_MULTIFILE ;
- generators.register-standard doxygen.html-redirect
- : DOXYGEN_HTML_MULTIFILE : DOXYGEN_HTML ;
- generators.register-standard doxygen.copy-latex-pngs
- : DOXYGEN_HTML : DOXYGEN_XML_IMAGES ;
-
- IMPORT $(__name__) : doxygen : : doxygen ;
- }
-
- if $(name)
- {
- modify-config ;
- .doxygen = $(name) ;
- check-doxygen ;
- }
-
- if ! $(.doxygen)
- {
- check-doxygen ;
- }
-}
-
-
-local rule freeze-config ( )
-{
- if ! $(.initialized)
- {
- import errors ;
- errors.user-error doxygen must be initialized before it can be used. ;
- }
- if ! $(.config-frozen)
- {
- .config-frozen = true ;
- if [ .is-cygwin ]
- {
- .is-cygwin = true ;
- }
- }
-}
-
-
-local rule modify-config ( )
-{
- if $(.config-frozen)
- {
- import errors ;
- errors.user-error "Cannot change doxygen after it has been used." ;
- }
-}
-
-
-local rule check-doxygen ( )
-{
- if --debug-configuration in [ modules.peek : ARGV ]
- {
- ECHO "notice:" using doxygen ":" $(.doxygen) ;
- }
- local extra-paths ;
- if [ os.name ] = NT
- {
- local ProgramFiles = [ modules.peek : ProgramFiles ] ;
- if $(ProgramFiles)
- {
- extra-paths = "$(ProgramFiles:J= )" ;
- }
- else
- {
- extra-paths = "C:\\Program Files" ;
- }
- }
- .doxygen = [ common.get-invocation-command doxygen : doxygen : $(.doxygen) :
- $(extra-paths) ] ;
-}
-
-
-rule name ( )
-{
- freeze-config ;
- return $(.doxygen) ;
-}
-
-
-local rule .is-cygwin ( )
-{
- if [ os.on-windows ]
- {
- local file = [ path.make [ modules.binding $(__name__) ] ] ;
- local dir = [ path.native [ path.join [ path.parent $(file) ] doxygen ]
- ] ;
- local command = cd \"$(dir)\" "&&" \"$(.doxygen)\"
- windows-paths-check.doxyfile 2>&1 ;
- command = $(command:J=" ") ;
- result = [ SHELL $(command) ] ;
- if [ MATCH "(Parsing file /)" : $(result) ]
- {
- return true ;
- }
- }
-}
-
-
-# Runs Doxygen on the given Doxygen configuration file (the source) to generate
-# the Doxygen files. The output is dumped according to the settings in the
-# Doxygen configuration file, not according to the target! Because of this, we
-# essentially "touch" the target file, in effect making it look like we have
-# really written something useful to it. Anyone that uses this action must deal
-# with this behavior.
-#
-actions doxygen-action
-{
- $(RM) "$(*.XML)" & "$(NAME:E=doxygen)" "$(>)" && echo "Stamped" > "$(<)"
-}
-
-
-# Runs the Python doxproc XML processor.
-#
-actions doxproc
-{
- python "$(DOXPROC)" "--xmldir=$(>)" "--output=$(<)" "$(OPTIONS)" "--id=$(ID)" "--title=$(TITLE)"
-}
-
-
-rule translate-path ( path )
-{
- freeze-config ;
- if [ os.on-windows ]
- {
- if [ os.name ] = CYGWIN
- {
- if $(.is-cygwin)
- {
- return $(path) ;
- }
- else
- {
- return $(path:W) ;
- }
- }
- else
- {
- if $(.is-cygwin)
- {
- match = [ MATCH ^(.):(.*) : $(path) ] ;
- if $(match)
- {
- return /cygdrive/$(match[1])$(match[2]:T) ;
- }
- else
- {
- return $(path:T) ;
- }
- }
- else
- {
- return $(path) ;
- }
- }
- }
- else
- {
- return $(path) ;
- }
-}
-
-
-# Generates a doxygen configuration file (doxyfile) given a set of C++ sources
-# and a property list that may contain <doxygen:param> features.
-#
-rule headers-to-doxyfile ( target : sources * : properties * )
-{
- local text = "# Generated by Boost.Build version 2" ;
-
- local output-dir ;
-
- # Translate <doxygen:param> into command line flags.
- for local param in [ feature.get-values <doxygen:param> : $(properties) ]
- {
- local namevalue = [ MATCH ([^=]*)=(.*) : $(param) ] ;
- if $(namevalue[1]) = OUTPUT_DIRECTORY
- {
- output-dir = [ translate-path [ utility.unquote $(namevalue[2]) ] ]
- ;
- text += "OUTPUT_DIRECTORY = \"$(output-dir)\"" ;
- }
- else
- {
- text += "$(namevalue[1]) = $(namevalue[2])" ;
- }
- }
-
- if ! $(output-dir)
- {
- output-dir = [ translate-path [ on $(target) return $(LOCATE) ] ] ;
- text += "OUTPUT_DIRECTORY = \"$(output-dir)\"" ;
- }
-
- local headers ;
- for local header in $(sources:G=)
- {
- header = [ translate-path $(header) ] ;
- headers += \"$(header)\" ;
- }
-
- # Doxygen generates LaTex by default. So disable it unconditionally, or at
- # least until someone needs, and hence writes support for, LaTex output.
- text += "GENERATE_LATEX = NO" ;
- text += "INPUT = $(headers:J= )" ;
- print.output $(target) plain ;
- print.text $(text) : true ;
-}
-
-
-# Run Doxygen. See doxygen-action for a description of the strange properties of
-# this rule.
-#
-rule run ( target : source : properties * )
-{
- freeze-config ;
- if <doxygen.rmdir>on in $(properties)
- {
- local output-dir = [ path.make [ MATCH
- <doxygen:param>OUTPUT_DIRECTORY=\"?([^\"]*) : $(properties) ] ] ;
- local html-dir = [ path.make [ MATCH <doxygen:param>HTML_OUTPUT=(.*) :
- $(properties) ] ] ;
- if $(output-dir) && $(html-dir) &&
- [ path.glob $(output-dir) : $(html-dir) ]
- {
- HTMLDIR on $(target) = [ path.native [ path.join $(output-dir)
- $(html-dir) ] ] ;
- rm-htmldir $(target) ;
- }
- }
- doxygen-action $(target) : $(source) ;
- NAME on $(target) = $(.doxygen) ;
- RM on $(target) = [ modules.peek common : RM ] ;
- *.XML on $(target) = [ path.native [ path.join [ path.make [ on $(target)
- return $(LOCATE) ] ] $(target:B:S=) *.xml ] ] ;
-}
-
-
-if [ os.name ] = NT
-{
- RMDIR = rmdir /s /q ;
-}
-else
-{
- RMDIR = rm -rf ;
-}
-
-actions quietly rm-htmldir
-{
- $(RMDIR) $(HTMLDIR)
-}
-
-
-# The rules below require BoostBook stylesheets, so we need some code to check
-# that the boostbook module has actualy been initialized.
-#
-rule check-boostbook ( )
-{
- if ! [ modules.peek boostbook : .initialized ]
- {
- import errors ;
- errors.user-error
- : The boostbook module is not initialized you have attempted to use
- : the 'doxygen' toolset, which requires BoostBook, but never
- : initialized BoostBook.
- : Hint: add 'using boostbook ;' to your user-config.jam. ;
- }
-}
-
-
-# Collect the set of Doxygen XML files into a single XML source file that can be
-# handled by an XSLT processor. The source is completely ignored (see
-# doxygen-action), because this action picks up the Doxygen XML index file xml/
-# index.xml. This is because we can not teach Doxygen to act like a NORMAL
-# program and take a "-o output.xml" argument (grrrr). The target of the
-# collection will be a single Doxygen XML file.
-#
-rule collect ( target : source : properties * )
-{
- check-boostbook ;
- local collect-xsl-dir
- = [ path.native [ path.join [ boostbook.xsl-dir ] doxygen collect ] ] ;
- local source-path
- = [ path.make [ on $(source) return $(LOCATE) ] ] ;
- local collect-path
- = [ path.root [ path.join $(source-path) $(source:B) ] [ path.pwd ] ] ;
- local native-path
- = [ path.native $(collect-path) ] ;
- local real-source
- = [ path.native [ path.join $(collect-path) index.xml ] ] ;
- xsltproc.xslt $(target) : $(real-source) $(collect-xsl-dir:S=.xsl)
- : <xsl:param>doxygen.xml.path=$(native-path) ;
-}
-
-
-# Translate Doxygen XML into BoostBook.
-#
-rule xml-to-boostbook ( target : source : properties * )
-{
- check-boostbook ;
- local xsl-dir = [ boostbook.xsl-dir ] ;
- local d2b-xsl = [ path.native [ path.join [ boostbook.xsl-dir ] doxygen
- doxygen2boostbook.xsl ] ] ;
-
- local xslt-properties = $(properties) ;
- for local prefix in [ feature.get-values <prefix> : $(properties) ]
- {
- xslt-properties += "<xsl:param>boost.doxygen.header.prefix=$(prefix)" ;
- }
- for local title in [ feature.get-values <reftitle> : $(properties) ]
- {
- xslt-properties += "<xsl:param>boost.doxygen.reftitle=$(title)" ;
- }
-
- xsltproc.xslt $(target) : $(source) $(d2b-xsl) : $(xslt-properties) ;
-}
-
-
-toolset.flags doxygen.xml-dir-to-boostbook OPTIONS <doxygen.doxproc.index>yes :
- --enable-index ;
-toolset.flags doxygen.xml-dir-to-boostbook ID <doxygen.doxproc.id> ;
-toolset.flags doxygen.xml-dir-to-boostbook TITLE <doxygen.doxproc.title> ;
-
-
-rule xml-dir-to-boostbook ( target : source : properties * )
-{
- DOXPROC on $(target) = $(.doxproc) ;
- LOCATE on $(source:S=) = [ on $(source) return $(LOCATE) ] ;
- doxygen.doxproc $(target) : $(source:S=) ;
-}
-
-
-# Generate the HTML redirect to HTML dir index.html file.
-#
-rule html-redirect ( target : source : properties * )
-{
- local uri = "$(target:B)/index.html" ;
- print.output $(target) plain ;
- print.text
-"<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\"
- \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">
-<html xmlns=\"http://www.w3.org/1999/xhtml\">
-<head>
- <meta http-equiv=\"refresh\" content=\"0; URL=$(uri)\" />
-
- <title></title>
-</head>
-
-<body>
- Automatic redirection failed, please go to <a href=
- \"$(uri)\">$(uri)</a>.
-</body>
-</html>
-"
- : true ;
-}
-
-rule copy-latex-pngs ( target : source : requirements * )
-{
- local directory = [ path.native [ feature.get-values <doxygen:xml-imagedir>
- : $(requirements) ] ] ;
- local location = [ on $(target) return $(LOCATE) ] ;
-
- local pdf-location = [ path.native [ path.join [ path.make $(location) ]
- [ path.make $(directory) ] ] ] ;
- local html-location = [ path.native [ path.join . html [ path.make
- $(directory) ] ] ] ;
-
- common.MkDir $(pdf-location) ;
- common.MkDir $(html-location) ;
-
- DEPENDS $(target) : $(pdf-location) $(html-location) ;
-
- if [ os.name ] = NT
- {
- CP on $(target) = copy /y ;
- FROM on $(target) = \\*.png ;
- TOHTML on $(target) = .\\html\\$(directory) ;
- TOPDF on $(target) = \\$(directory) ;
- }
- else
- {
- CP on $(target) = cp ;
- FROM on $(target) = /*.png ;
- TOHTML on $(target) = ./html/$(directory) ;
- TOPDF on $(target) = $(target:D)/$(directory) ;
- }
-}
-
-actions copy-latex-pngs
-{
- $(CP) $(>:S=)$(FROM) $(TOHTML)
- $(CP) $(>:S=)$(FROM) $(<:D)$(TOPDF)
- echo "Stamped" > "$(<)"
-}
-
-
-# Building latex images for doxygen XML depends on latex, dvips, and gs being in
-# your PATH. This is true for most Unix installs, but not on Win32, where you
-# will need to install MkTex and Ghostscript and add these tools to your path.
-
-actions check-latex
-{
- latex -version >$(<)
-}
-
-actions check-dvips
-{
- dvips -version >$(<)
-}
-
-if [ os.name ] = "NT"
-{
- actions check-gs
- {
- gswin32c -version >$(<)
- }
-}
-else
-{
- actions check-gs
- {
- gs -version >$(<)
- }
-}
-
-
-local rule check-tools-targets ( project )
-{
- if ! $(.check-tools-targets)
- {
- # Find the root project.
- #
- # This is a best effort attempt to avoid using different locations for
- # storing *.check files depending on which project imported the doxygen
- # toolset first. The files are stored in a location related to the
- # project's root project. Note that this location may change depending
- # on the folder the build was run from in case the build uses multiple
- # related projects with their own Jamroot separate modules.
- local project-module = [ $(project).project-module ] ;
- local root-module = [ project.get-jamroot-module $(project-module) ] ;
- if ! $(root-module)
- {
- import errors ;
- if [ project.is-config-module $(project-module) ]
- {
- errors.user-error doxygen targets can not be declared in Boost
- Build's configuration modules. ;
- }
- else
- {
- errors.user-error doxygen targets can not be declared in
- standalone projects. : use a Jamfile/Jamroot project
- instead. ;
- }
- }
- local root-project = [ project.target $(root-module) ] ;
-
- local targets =
- [ new file-target latex.check : : $(root-project) : [ new action :
- doxygen.check-latex ] ]
- [ new file-target dvips.check : : $(root-project) : [ new action :
- doxygen.check-dvips ] ]
- [ new file-target gs.check : : $(root-project) : [ new action :
- doxygen.check-gs ] ] ;
-
- for local target in $(targets)
- {
- .check-tools-targets += [ virtual-target.register $(target) ] ;
- }
- }
- return $(.check-tools-targets) ;
-}
-
-
-project.initialize $(__name__) ;
-project doxygen ;
-
-class doxygen-check-tools-target-class : basic-target
-{
- rule construct ( name : sources * : property-set )
- {
- IMPORT doxygen : check-tools-targets : $(__name__) :
- doxygen.check-tools-targets ;
- return [ property-set.empty ] [ doxygen.check-tools-targets [ project ]
- ] ;
- }
-}
-
-
-# Declares a metatarget for collecting version information on different external
-# tools used in this module.
-#
-rule check-tools ( target )
-{
- freeze-config ;
- targets.create-metatarget doxygen-check-tools-target-class :
- [ project.current ] : $(target) ;
-}
-
-
-# User-level rule to generate HTML files or BoostBook XML from a set of headers
-# via Doxygen.
-#
-rule doxygen ( target : sources + : requirements * : default-build * :
- usage-requirements * )
-{
- freeze-config ;
- local project = [ project.current ] ;
-
- if $(target:S) = .html
- {
- # Build an HTML directory from the sources.
- local html-location = [ feature.get-values <location> : $(requirements)
- ] ;
- local output-dir ;
- if [ $(project).get build-dir ]
- {
- # Explicitly specified build dir. Add html at the end.
- output-dir = [ path.join [ $(project).build-dir ]
- $(html-location:E=html) ] ;
- }
- else
- {
- # Trim 'bin' from implicit build dir, for no other reason than
- # backward compatibility.
- output-dir = [ path.join [ path.parent [ $(project).build-dir ] ]
- $(html-location:E=html) ] ;
- }
- output-dir = [ path.root $(output-dir) [ path.pwd ] ] ;
- local output-dir-native = [ path.native $(output-dir) ] ;
- requirements = [ property.change $(requirements) : <location> ] ;
-
- # The doxygen configuration file.
- targets.create-typed-target DOXYFILE : $(project) : $(target:S=.tag)
- : $(sources)
- : $(requirements)
- <doxygen:param>GENERATE_HTML=YES
- <doxygen:param>GENERATE_XML=NO
- <doxygen:param>"OUTPUT_DIRECTORY=\"$(output-dir-native)\""
- <doxygen:param>HTML_OUTPUT=$(target:B)
- : $(default-build) ;
- $(project).mark-target-as-explicit $(target:S=.tag) ;
-
- # The html directory to generate by running doxygen.
- targets.create-typed-target DOXYGEN_HTML_MULTIFILE : $(project)
- : $(target:S=.dir) # Name.
- : $(target:S=.tag) # Sources.
- : $(requirements)
- <doxygen:param>"OUTPUT_DIRECTORY=\"$(output-dir-native)\""
- <doxygen:param>HTML_OUTPUT=$(target:B)
- : $(default-build) ;
- $(project).mark-target-as-explicit $(target:S=.dir) ;
-
- # The redirect html file into the generated html.
- targets.create-typed-target DOXYGEN_HTML : $(project) : $(target)
- : $(target:S=.dir) # Sources.
- : $(requirements) <location>$(output-dir)
- : $(default-build) ;
- }
- else
- {
- # Build a BoostBook XML file from the sources.
- local location-xml = [ feature.get-values <location> : $(requirements) ]
- ;
- requirements = [ property.change $(requirements) : <location> ] ;
- local target-xml = $(target:B=$(target:B)-xml) ;
-
- # Check whether we need to build images.
- local images-location = [ feature.get-values <doxygen:xml-imagedir> :
- $(requirements) ] ;
- if $(images-location)
- {
- # Prepare a metatarget for collecting used external tool version
- # information. We use only one such metatarget as they always
- # produce the same files and we do not want to deal with multiple
- # metatargets having matching names, causing 'ambiguous variants'
- # errors.
- if ! $(.check-tools)
- {
- # FIXME: Since we have the check-tools target object reference,
- # see how we can use that instead of having to construct a valid
- # target reference string for use in <dependency> property
- # values.
- local project-id = --doxygen.check-tools-project-- ;
- local target-id = --doxygen.check-tools-- ;
- local pm = [ $(project).project-module ] ;
- project.register-id $(project-id) : $(pm) ;
- check-tools $(target-id) ;
- .check-tools = /$(project-id)//$(target-id) ;
- }
-
- doxygen $(target).doxygen-xml-images.html : $(sources) :
- $(requirements)
- <doxygen.rmdir>on
- <doxygen:param>QUIET=YES
- <doxygen:param>WARNINGS=NO
- <doxygen:param>WARN_IF_UNDOCUMENTED=NO
- <dependency>$(.check-tools) ;
- $(project).mark-target-as-explicit $(target).doxygen-xml-images.html
- ;
-
- targets.create-typed-target DOXYGEN_XML_IMAGES : $(project)
- : $(target).doxygen-xml-images # Name.
- : $(target).doxygen-xml-images.html # Sources.
- : $(requirements)
- : $(default-build) ;
- $(project).mark-target-as-explicit $(target).doxygen-xml-images ;
-
- if ! [ MATCH (/)$ : $(images-location) ]
- {
- images-location = $(images-location)/ ;
- }
-
- requirements +=
- <dependency>$(target).doxygen-xml-images
- <xsl:param>boost.doxygen.formuladir=$(images-location) ;
- }
-
- # The doxygen configuration file.
- targets.create-typed-target DOXYFILE : $(project) : $(target-xml:S=.tag)
- : $(sources)
- : $(requirements)
- <doxygen:param>GENERATE_HTML=NO
- <doxygen:param>GENERATE_XML=YES
- <doxygen:param>XML_OUTPUT=$(target-xml)
- : $(default-build) ;
- $(project).mark-target-as-explicit $(target-xml:S=.tag) ;
-
- # The Doxygen XML directory for the processed source files.
- targets.create-typed-target DOXYGEN_XML_MULTIFILE : $(project)
- : $(target-xml:S=.dir) # Name.
- : $(target-xml:S=.tag) # Sources.
- : $(requirements)
- : $(default-build) ;
- $(project).mark-target-as-explicit $(target-xml:S=.dir) ;
-
- # The resulting BoostBook file is generated by the processor tool. The
- # tool can be either the xsltproc plus accompanying XSL scripts. Or it
- # can be the python doxproc.py script.
- targets.create-typed-target BOOSTBOOK : $(project) : $(target-xml)
- : $(target-xml:S=.dir) # Sources.
- : $(requirements)
- : $(default-build) ;
- $(project).mark-target-as-explicit $(target-xml) ;
-
- stage $(target:S=.xml) # Name.
- : $(target-xml) # Sources.
- : $(requirements)
- <location>$(location-xml:E=.)
- <name>$(target:S=.xml)
- : $(default-build) ;
- $(project).mark-target-as-explicit $(target:S=.xml) ;
-
- # TODO: See why this alias target is used here instead of simply naming
- # the previous stage target $(target) and having it specify the alias
- # target's usage requirements directly.
- alias $(target) : : $(requirements) : $(default-build) :
- $(usage-requirements) <dependency>$(target:S=.xml) ;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/doxygen/windows-paths-check.doxyfile
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/doxygen/windows-paths-check.doxyfile b/ext/kenlm/jam-files/boost-build/tools/doxygen/windows-paths-check.doxyfile
deleted file mode 100644
index 9b969df..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/doxygen/windows-paths-check.doxyfile
+++ /dev/null
@@ -1,3 +0,0 @@
-INPUT = windows-paths-check.hpp
-GENERATE_HTML = NO
-GENERATE_LATEX = NO
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/doxygen/windows-paths-check.hpp
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/doxygen/windows-paths-check.hpp b/ext/kenlm/jam-files/boost-build/tools/doxygen/windows-paths-check.hpp
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/fop.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/fop.jam b/ext/kenlm/jam-files/boost-build/tools/fop.jam
deleted file mode 100644
index c24b872..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/fop.jam
+++ /dev/null
@@ -1,69 +0,0 @@
-# Copyright (C) 2003-2004 Doug Gregor and Dave Abrahams. Distributed
-# under the Boost Software License, Version 1.0. (See accompanying
-# file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-#
-# This module defines rules to handle generation of PDF and
-# PostScript files from XSL Formatting Objects via Apache FOP
-
-import generators ;
-import common ;
-import boostbook ;
-
-generators.register-standard fop.render.pdf : FO : PDF ;
-generators.register-standard fop.render.ps : FO : PS ;
-
-# Initializes the fop toolset.
-#
-rule init ( fop-command ? : java-home ? : java ? )
-{
- local has-command = $(.has-command) ;
-
- if $(fop-command)
- {
- .has-command = true ;
- }
-
- if $(fop-command) || ! $(has-command)
- {
- fop-command = [ common.get-invocation-command fop : fop : $(fop-command)
- : [ modules.peek : FOP_DIR ] ] ;
- }
-
- if $(fop-command)
- {
- .FOP_COMMAND = $(fop-command) ;
- }
-
- if $(java-home) || $(java)
- {
- .FOP_SETUP = ;
-
-
- # JAVA_HOME is the location that java was installed to.
-
- if $(java-home)
- {
- .FOP_SETUP += [ common.variable-setting-command JAVA_HOME : $(java-home) ] ;
- }
-
- # JAVACMD is the location that of the java executable, useful for a
- # non-standard java installation, where the executable isn't at
- # $JAVA_HOME/bin/java.
-
- if $(java)
- {
- .FOP_SETUP += [ common.variable-setting-command JAVACMD : $(java) ] ;
- }
- }
-}
-
-actions render.pdf
-{
- $(.FOP_SETUP) $(.FOP_COMMAND:E=fop) $(>) $(<)
-}
-
-actions render.ps
-{
- $(.FOP_SETUP) $(.FOP_COMMAND:E=fop) $(>) -ps $(<)
-}
[35/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/qt4.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/qt4.jam b/ext/kenlm/jam-files/boost-build/tools/qt4.jam
deleted file mode 100644
index a3aac61..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/qt4.jam
+++ /dev/null
@@ -1,755 +0,0 @@
-# Copyright 2002-2006 Vladimir Prus
-# Copyright 2005 Alo Sarv
-# Copyright 2005-2009 Juergen Hunold
-#
-# Distributed under the Boost Software License, Version 1.0. (See
-# accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# Qt4 library support module
-#
-# The module attempts to auto-detect QT installation location from QTDIR
-# environment variable; failing that, installation location can be passed as
-# argument:
-#
-# toolset.using qt4 : /usr/local/Trolltech/Qt-4.0.0 ;
-#
-# The module supports code generation from .ui and .qrc files, as well as
-# running the moc preprocessor on headers. Note that you must list all your
-# moc-able headers in sources.
-#
-# Example:
-#
-# exe myapp : myapp.cpp myapp.h myapp.ui myapp.qrc
-# /qt4//QtGui /qt4//QtNetwork ;
-#
-# It's also possible to run moc on cpp sources:
-#
-# import cast ;
-#
-# exe myapp : myapp.cpp [ cast _ moccable-cpp : myapp.cpp ] /qt4//QtGui ;
-#
-# When moccing source file myapp.cpp you need to include "myapp.moc" from
-# myapp.cpp. When moccing .h files, the output of moc will be automatically
-# compiled and linked in, you don't need any includes.
-#
-# This is consistent with Qt guidelines:
-# http://qt-project.org/doc/qt-4.8/moc.html
-#
-# The .qrc processing utility supports various command line option (see
-# http://qt-project.org/doc/qt-4.8/rcc.html for a complete list). The
-# module provides default arguments for the "output file" and
-# "initialization function name" options. Other options can be set through
-# the <rccflags> build property. E.g. if you wish the compression settings
-# to be more aggressive than the defaults, you can apply them too all .qrc
-# files like this:
-#
-# project my-qt-project :
-# requirements
-# <rccflags>"-compress 9 -threshold 10"
-# ;
-#
-# Of course, this property can also be specified on individual targets.
-
-
-import modules ;
-import feature ;
-import errors ;
-import type ;
-import "class" : new ;
-import generators ;
-import project ;
-import toolset : flags ;
-import os ;
-import virtual-target ;
-import scanner ;
-
-# Qt3Support control feature
-#
-# Qt4 configure defaults to build Qt4 libraries with Qt3Support.
-# The autodetection is missing, so we default to disable Qt3Support.
-# This prevents the user from inadvertedly using a deprecated API.
-#
-# The Qt3Support library can be activated by adding
-# "<qt3support>on" to requirements
-#
-# Use "<qt3support>on:<define>QT3_SUPPORT_WARNINGS"
-# to get warnings about deprecated Qt3 support funtions and classes.
-# Files ported by the "qt3to4" conversion tool contain _tons_ of
-# warnings, so this define is not set as default.
-#
-# Todo: Detect Qt3Support from Qt's configure data.
-# Or add more auto-configuration (like python).
-feature.feature qt3support : off on : propagated link-incompatible ;
-
-# The Qt version used for requirements
-# Valid are <qt>4.4 or <qt>4.5.0
-# Auto-detection via qmake sets '<qt>major.minor.patch'
-feature.feature qt : : propagated ;
-
-# Extra flags for rcc
-feature.feature rccflags : : free ;
-
-project.initialize $(__name__) ;
-project qt ;
-
-# Save the project so that we tolerate 'import + using' combo.
-.project = [ project.current ] ;
-
-# Helper utils for easy debug output
-if [ MATCH (--debug-configuration) : [ modules.peek : ARGV ] ]
-{
- .debug-configuration = TRUE ;
-}
-
-local rule debug-message ( message * )
-{
- if $(.debug-configuration) = TRUE
- {
- ECHO notice: [qt4-cfg] $(message) ;
- }
-}
-
-# Capture qmake output line by line
-local rule read-output ( content )
-{
- local lines ;
- local nl = "
-" ;
- local << = "([^$(nl)]*)[$(nl)](.*)" ;
- local line+ = [ MATCH "$(<<)" : "$(content)" ] ;
- while $(line+)
- {
- lines += $(line+[1]) ;
- line+ = [ MATCH "$(<<)" : "$(line+[2])" ] ;
- }
- return $(lines) ;
-}
-
-# Capture Qt version from qmake
-local rule check-version ( bin_prefix )
-{
- full-cmd = $(bin_prefix)"/qmake -v" ;
- debug-message Running '$(full-cmd)' ;
- local output = [ SHELL $(full-cmd) ] ;
- for line in [ read-output $(output) ]
- {
- # Parse the output to get all the results.
- if [ MATCH "QMake" : $(line) ]
- {
- # Skip first line of output
- }
- else
- {
- temp = [ MATCH "([0-9]*)\\.([0-9]*)\\.([0-9]*)" : $(line) ] ;
- }
- }
- return $(temp) ;
-}
-
-# Validate the version string and extract the major/minor part we care about.
-#
-local rule split-version ( version )
-{
- local major-minor = [ MATCH ^([0-9]+)\.([0-9]+)(.*)$ : $(version) : 1 2 3 ] ;
- if ! $(major-minor[2]) || $(major-minor[3])
- {
- ECHO "Warning: 'using qt' expects a two part (major, minor) version number; got" $(version) instead ;
-
- # Add a zero to account for the missing digit if necessary.
- major-minor += 0 ;
- }
-
- return $(major-minor[1]) $(major-minor[2]) ;
-}
-
-# Initialize the QT support module.
-# Parameters:
-# - 'prefix' parameter tells where Qt is installed.
-# - 'full_bin' optional full path to Qt binaries (qmake,moc,uic,rcc)
-# - 'full_inc' optional full path to Qt top-level include directory
-# - 'full_lib' optional full path to Qt library directory
-# - 'version' optional version of Qt, else autodetected via 'qmake -v'
-# - 'condition' optional requirements
-rule init ( prefix : full_bin ? : full_inc ? : full_lib ? : version ? : condition * )
-{
- project.push-current $(.project) ;
-
- debug-message "==== Configuring Qt ... ====" ;
- for local v in version cmd-or-prefix includes libraries condition
- {
- if $($(v))
- {
- debug-message " user-specified "$(v): '$($(v))' ;
- }
- }
-
- # Needed as default value
- .prefix = $(prefix) ;
-
- # pre-build paths to detect reinitializations changes
- local inc_prefix lib_prefix bin_prefix ;
- if $(full_inc)
- {
- inc_prefix = $(full_inc) ;
- }
- else
- {
- inc_prefix = $(prefix)/include ;
- }
- if $(full_lib)
- {
- lib_prefix = $(full_lib) ;
- }
- else
- {
- lib_prefix = $(prefix)/lib ;
- }
- if $(full_bin)
- {
- bin_prefix = $(full_bin) ;
- }
- else
- {
- bin_prefix = $(prefix)/bin ;
- }
-
- # Globally needed variables
- .incprefix = $(inc_prefix) ;
- .libprefix = $(lib_prefix) ;
- .binprefix = $(bin_prefix) ;
-
- if ! $(.initialized)
- {
- # Make sure this is initialised only once
- .initialized = true ;
-
- # Generates cpp files from header files using "moc" tool
- generators.register-standard qt4.moc : H : CPP(moc_%) : <allow>qt4 ;
-
- # The OBJ result type is a fake, 'H' will be really produced. See
- # comments on the generator class, defined below the 'init' function.
- generators.register [ new uic-generator qt4.uic : UI : OBJ :
- <allow>qt4 ] ;
-
- # The OBJ result type is a fake here too.
- generators.register [ new moc-h-generator
- qt4.moc.inc : MOCCABLE_CPP : OBJ : <allow>qt4 ] ;
-
- generators.register [ new moc-inc-generator
- qt4.moc.inc : MOCCABLE_H : OBJ : <allow>qt4 ] ;
-
- # Generates .cpp files from .qrc files.
- generators.register-standard qt4.rcc : QRC : CPP(qrc_%) : <allow>qt4 ;
-
- # dependency scanner for wrapped files.
- type.set-scanner QRC : qrc-scanner ;
-
- # Save value of first occuring prefix
- .PREFIX = $(prefix) ;
- }
-
- if $(version)
- {
- major-minor = [ split-version $(version) ] ;
- version = $(major-minor:J=.) ;
- }
- else
- {
- version = [ check-version $(bin_prefix) ] ;
- if $(version)
- {
- version = $(version:J=.) ;
- }
- debug-message Detected version '$(version)' ;
- }
-
- local target-requirements = $(condition) ;
-
- # Add the version, if any, to the target requirements.
- if $(version)
- {
- if ! $(version) in [ feature.values qt ]
- {
- feature.extend qt : $(version) ;
- }
- target-requirements += <qt>$(version:E=default) ;
- }
-
- local target-os = [ feature.get-values target-os : $(condition) ] ;
- if ! $(target-os)
- {
- target-os ?= [ feature.defaults target-os ] ;
- target-os = $(target-os:G=) ;
- target-requirements += <target-os>$(target-os) ;
- }
-
- # Build exact requirements for the tools
- local tools-requirements = $(target-requirements:J=/) ;
-
- debug-message "Details of this Qt configuration:" ;
- debug-message " prefix: " '$(prefix:E=<empty>)' ;
- debug-message " binary path: " '$(bin_prefix:E=<empty>)' ;
- debug-message " include path:" '$(inc_prefix:E=<empty>)' ;
- debug-message " library path:" '$(lib_prefix:E=<empty>)' ;
- debug-message " target requirements:" '$(target-requirements)' ;
- debug-message " tool requirements: " '$(tools-requirements)' ;
-
- # setup the paths for the tools
- toolset.flags qt4.moc .BINPREFIX $(tools-requirements) : $(bin_prefix) ;
- toolset.flags qt4.rcc .BINPREFIX $(tools-requirements) : $(bin_prefix) ;
- toolset.flags qt4.uic .BINPREFIX $(tools-requirements) : $(bin_prefix) ;
-
- # TODO: 2009-02-12: Better support for directories
- # Most likely needed are separate getters for: include,libraries,binaries and sources.
- toolset.flags qt4.directory .PREFIX $(tools-requirements) : $(prefix) ;
-
- # Test for a buildable Qt.
- if [ glob $(.prefix)/Jamroot ]
- {
- .bjam-qt = true
-
- # this will declare QtCore (and qtmain on <target-os>windows)
- add-shared-library QtCore ;
- }
- else
- # Setup common pre-built Qt.
- # Special setup for QtCore on which everything depends
- {
- local link = [ feature.get-values link : $(condition) ] ;
-
- local usage-requirements =
- <include>$(.incprefix)
- <library-path>$(.libprefix)
- <threading>multi
- <allow>qt4 ;
-
- if $(link) in shared
- {
- usage-requirements += <dll-path>$(.libprefix) ;
- }
-
- local suffix ;
-
- # Since Qt-4.2, debug versions on unix have to be built
- # separately and therefore have no suffix.
- .suffix_version = "" ;
- .suffix_debug = "" ;
-
- # Control flag for auto-configuration of the debug libraries.
- # This setup requires Qt 'configure -debug-and-release'.
- # Only available on some platforms.
- # ToDo: 2009-02-12: Maybe throw this away and
- # require separate setup with <variant>debug as condition.
- .have_separate_debug = FALSE ;
-
- # Setup other platforms
- if $(target-os) in windows cygwin
- {
- .have_separate_debug = TRUE ;
-
- # On NT, the shared libs have "4" suffix, and "d" suffix in debug builds.
- if $(link) in shared
- {
- .suffix_version = "4" ;
- }
- .suffix_debug = "d" ;
-
- # On Windows we must link against the qtmain library
- lib qtmain
- : # sources
- : # requirements
- <name>qtmain$(.suffix_debug)
- <variant>debug
- $(target-requirements)
- ;
-
- lib qtmain
- : # sources
- : # requirements
- <name>qtmain
- $(target-requirements)
- ;
- }
- else if $(target-os) = darwin
- {
- # On MacOS X, both debug and release libraries are available.
- .suffix_debug = "_debug" ;
-
- .have_separate_debug = TRUE ;
-
- alias qtmain ;
- }
- else
- {
- alias qtmain : : $(target-requirements) ;
- }
-
- lib QtCore : qtmain
- : # requirements
- <name>QtCore$(.suffix_version)
- $(target-requirements)
- : # default-build
- : # usage-requirements
- <define>QT_CORE_LIB
- <define>QT_NO_DEBUG
- <include>$(.incprefix)/QtCore
- $(usage-requirements)
- ;
-
- if $(.have_separate_debug) = TRUE
- {
- debug-message Configure debug libraries with suffix '$(.suffix_debug)' ;
-
- lib QtCore : $(main)
- : # requirements
- <name>QtCore$(.suffix_debug)$(.suffix_version)
- <variant>debug
- $(target-requirements)
- : # default-build
- : # usage-requirements
- <define>QT_CORE_LIB
- <include>$(.incprefix)/QtCore
- $(usage-requirements)
- ;
- }
- }
-
- # Initialising the remaining libraries is canonical
- # parameters 'module' : 'depends-on' : 'usage-define' : 'requirements' : 'include'
- # 'include' only for non-canonical include paths.
- add-shared-library QtGui : QtCore : QT_GUI_LIB : $(target-requirements) ;
- add-shared-library QtNetwork : QtCore : QT_NETWORK_LIB : $(target-requirements) ;
- add-shared-library QtSql : QtCore : QT_SQL_LIB : $(target-requirements) ;
- add-shared-library QtXml : QtCore : QT_XML_LIB : $(target-requirements) ;
-
- add-shared-library Qt3Support : QtGui QtNetwork QtXml QtSql
- : QT_QT3SUPPORT_LIB QT3_SUPPORT
- : <qt3support>on $(target-requirements) ;
-
- # Dummy target to enable "<qt3support>off" and
- # "<library>/qt//Qt3Support" at the same time. This enables quick
- # switching from one to the other for test/porting purposes.
- alias Qt3Support : : <qt3support>off $(target-requirements) ;
-
- # OpenGl Support
- add-shared-library QtOpenGL : QtGui : QT_OPENGL_LIB : $(target-requirements) ;
-
- # SVG-Support (Qt 4.1)
- add-shared-library QtSvg : QtXml QtOpenGL : QT_SVG_LIB : $(target-requirements) ;
-
- # Test-Support (Qt 4.1)
- add-shared-library QtTest : QtCore : : $(target-requirements) ;
-
- # Qt designer library
- add-shared-library QtDesigner : QtGui QtXml : : $(target-requirements) ;
- add-shared-library QtDesignerComponents : QtGui QtXml : : $(target-requirements) ;
-
- # Support for dynamic Widgets (Qt 4.1)
- add-static-library QtUiTools : QtGui QtXml : $(target-requirements) ;
-
- # DBus-Support (Qt 4.2)
- add-shared-library QtDBus : QtXml : : $(target-requirements) ;
-
- # Script-Engine (Qt 4.3)
- add-shared-library QtScript : QtGui QtXml : QT_SCRIPT_LIB : $(target-requirements) ;
-
- # Tools for the Script-Engine (Qt 4.5)
- add-shared-library QtScriptTools : QtScript : QT_SCRIPTTOOLS_LIB : $(target-requirements) ;
-
- # WebKit (Qt 4.4)
- add-shared-library QtWebKit : QtGui : QT_WEBKIT_LIB : $(target-requirements) ;
-
- # Phonon Multimedia (Qt 4.4)
- add-shared-library phonon : QtGui QtXml : QT_PHONON_LIB : $(target-requirements) ;
-
- # Multimedia engine (Qt 4.6)
- add-shared-library QtMultimedia : QtGui : QT_MULTIMEDIA_LIB : $(target-requirements) ;
-
- # XmlPatterns-Engine (Qt 4.4)
- add-shared-library QtXmlPatterns : QtNetwork : QT_XMLPATTERNS_LIB : $(target-requirements) ;
-
- # Help-Engine (Qt 4.4)
- add-shared-library QtHelp : QtGui QtSql QtXml : : $(target-requirements) ;
- add-shared-library QtCLucene : QCore QtSql QtXml : : $(target-requirements) ;
-
- # QML-Engine (Qt 4.7)
- add-shared-library QtDeclarative : QtGui QtXml : : $(target-requirements) ;
-
- # AssistantClient Support
- # Compat library removed in 4.7.0
- # Pre-4.4 help system, use QtHelp for new programs
- if $(version) < "4.7"
- {
- add-shared-library QtAssistantClient : QtGui : : $(target-requirements) : QtAssistant ;
- }
- debug-message "==== Configured Qt-$(version) ====" ;
-
- project.pop-current ;
-}
-
-rule initialized ( )
-{
- return $(.initialized) ;
-}
-
-
-
-# This custom generator is needed because in QT4, UI files are translated only
-# into H files, and no C++ files are created. Further, the H files need not be
-# passed via MOC. The header is used only via inclusion. If we define a standard
-# UI -> H generator, Boost.Build will run MOC on H, and then compile the
-# resulting cpp. It will give a warning, since output from moc will be empty.
-#
-# This generator is declared with a UI -> OBJ signature, so it gets invoked when
-# linking generator tries to convert sources to OBJ, but it produces target of
-# type H. This is non-standard, but allowed. That header won't be mocced.
-#
-class uic-generator : generator
-{
- rule __init__ ( * : * )
- {
- generator.__init__ $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ;
- }
-
- rule run ( project name ? : property-set : sources * )
- {
- if ! $(name)
- {
- name = [ $(sources[0]).name ] ;
- name = $(name:B) ;
- }
-
- local a = [ new action $(sources[1]) : qt4.uic : $(property-set) ] ;
-
- # The 'ui_' prefix is to match qmake's default behavior.
- local target = [ new file-target ui_$(name) : H : $(project) : $(a) ] ;
-
- local r = [ virtual-target.register $(target) ] ;
-
- # Since this generator will return a H target, the linking generator
- # won't use it at all, and won't set any dependency on it. However, we
- # need the target to be seen by bjam, so that dependency from sources to
- # this generated header is detected -- if jam does not know about this
- # target, it won't do anything.
- DEPENDS all : [ $(r).actualize ] ;
-
- return $(r) ;
- }
-}
-
-
-class moc-h-generator : generator
-{
- rule __init__ ( * : * )
- {
- generator.__init__ $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ;
- }
-
- rule run ( project name ? : property-set : sources * )
- {
- if ! $(sources[2]) && [ $(sources[1]).type ] = MOCCABLE_CPP
- {
- name = [ $(sources[0]).name ] ;
- name = $(name:B) ;
-
- local a = [ new action $(sources[1]) : qt4.moc.inc :
- $(property-set) ] ;
-
- local target = [ new file-target $(name) : MOC : $(project) : $(a)
- ] ;
-
- local r = [ virtual-target.register $(target) ] ;
-
- # Since this generator will return a H target, the linking generator
- # won't use it at all, and won't set any dependency on it. However,
- # we need the target to be seen by bjam, so that dependency from
- # sources to this generated header is detected -- if jam does not
- # know about this target, it won't do anything.
- DEPENDS all : [ $(r).actualize ] ;
-
- return $(r) ;
- }
- }
-}
-
-
-class moc-inc-generator : generator
-{
- rule __init__ ( * : * )
- {
- generator.__init__ $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ;
- }
-
- rule run ( project name ? : property-set : sources * )
- {
- if ! $(sources[2]) && [ $(sources[1]).type ] = MOCCABLE_H
- {
- name = [ $(sources[0]).name ] ;
- name = $(name:B) ;
-
- local a = [ new action $(sources[1]) : qt4.moc.inc :
- $(property-set) ] ;
-
- local target = [ new file-target moc_$(name) : CPP : $(project) :
- $(a) ] ;
-
- # Since this generator will return a H target, the linking generator
- # won't use it at all, and won't set any dependency on it. However,
- # we need the target to be seen by bjam, so that dependency from
- # sources to this generated header is detected -- if jam does not
- # know about this target, it won't do anything.
- DEPENDS all : [ $(target).actualize ] ;
-
- return [ virtual-target.register $(target) ] ;
- }
- }
-}
-
-
-# Query the installation directory. This is needed in at least two scenarios.
-# First, when re-using sources from the Qt-Tree. Second, to "install" custom Qt
-# plugins to the Qt-Tree.
-#
-rule directory
-{
- return $(.PREFIX) ;
-}
-
-# Add a shared Qt library.
-rule add-shared-library ( lib-name : depends-on * : usage-defines * : requirements * : include ? )
-{
- add-library $(lib-name) : $(.suffix_version) : $(depends-on) : $(usage-defines) : $(requirements) : $(include) ;
-}
-
-# Add a static Qt library.
-rule add-static-library ( lib-name : depends-on * : usage-defines * : requirements * : include ? )
-{
- add-library $(lib-name) : : $(depends-on) : $(usage-defines) : $(requirements) : $(include) ;
-}
-
-# Add a Qt library.
-# Static libs are unversioned, whereas shared libs have the major number as suffix.
-# Creates both release and debug versions on platforms where both are enabled by Qt configure.
-# Flags:
-# - lib-name Qt library Name
-# - version Qt major number used as shared library suffix (QtCore4.so)
-# - depends-on other Qt libraries
-# - usage-defines those are set by qmake, so set them when using this library
-# - requirements addional requirements
-# - include non-canonical include path. The canonical path is $(.incprefix)/$(lib-name).
-rule add-library ( lib-name : version ? : depends-on * : usage-defines * : requirements * : include ? )
-{
- if $(.bjam-qt)
- {
- # Import Qt module
- # Eveything will be setup there
- alias $(lib-name)
- : $(.prefix)//$(lib-name)
- :
- :
- : <allow>qt4 ;
- }
- else
- {
- local real_include ;
- real_include ?= $(include) ;
- real_include ?= $(lib-name) ;
-
- lib $(lib-name)
- : # sources
- $(depends-on)
- : # requirements
- <name>$(lib-name)$(version)
- $(requirements)
- : # default-build
- : # usage-requirements
- <define>$(usage-defines)
- <include>$(.incprefix)/$(real_include)
- ;
-
- if $(.have_separate_debug) = TRUE
- {
- lib $(lib-name)
- : # sources
- $(depends-on)
- : # requirements
- <name>$(lib-name)$(.suffix_debug)$(version)
- $(requirements)
- <variant>debug
- : # default-build
- : # usage-requirements
- <define>$(usage-defines)
- <include>$(.incprefix)/$(real_include)
- ;
- }
- }
-
- # Make library explicit so that a simple <use>qt4 will not bring in everything.
- # And some components like QtDBus/Phonon may not be available on all platforms.
- explicit $(lib-name) ;
-}
-
-# Use $(.BINPREFIX[-1]) for the paths as several tools-requirements can match.
-# The exact match is the last one.
-
-# Get <include> and <defines> from current toolset.
-flags qt4.moc INCLUDES <include> ;
-flags qt4.moc DEFINES <define> ;
-
-# need a newline for expansion of DEFINES and INCLUDES in the response file.
-.nl = "
-" ;
-
-# Processes headers to create Qt MetaObject information. Qt4-moc has its
-# c++-parser, so pass INCLUDES and DEFINES.
-# We use response file with one INCLUDE/DEFINE per line
-#
-actions moc
-{
- $(.BINPREFIX[-1])/moc -f $(>) -o $(<) @"@($(<).rsp:E=-D$(DEFINES)$(.nl) -I$(INCLUDES:T)$(.nl))"
-}
-
-# When moccing files for include only, we don't need -f, otherwise the generated
-# code will include the .cpp and we'll get duplicated symbols.
-#
-actions moc.inc
-{
- $(.BINPREFIX[-1])/moc $(>) -o $(<) @"@($(<).rsp:E=-D$(DEFINES)$(.nl) -I$(INCLUDES:T)$(.nl))"
-}
-
-
-# Get extra options for RCC
-flags qt4.rcc RCC_OPTIONS <rccflags> ;
-
-# Generates source files from resource files.
-#
-actions rcc
-{
- $(.BINPREFIX[-1])/rcc $(>) -name $(>:B) $(RCC_OPTIONS) -o $(<)
-}
-
-
-# Generates user-interface source from .ui files.
-#
-actions uic
-{
- $(.BINPREFIX[-1])/uic $(>) -o $(<)
-}
-
-
-# Scanner for .qrc files. Look for the CDATA section of the <file> tag. Ignore
-# the "alias" attribute. See http://doc.trolltech.com/qt/resources.html for
-# detailed documentation of the Qt Resource System.
-#
-class qrc-scanner : common-scanner
-{
- rule pattern ( )
- {
- return "<file.*>(.*)</file>" ;
- }
-}
-
-
-# Wrapped files are "included".
-scanner.register qrc-scanner : include ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/qt5.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/qt5.jam b/ext/kenlm/jam-files/boost-build/tools/qt5.jam
deleted file mode 100644
index a5fdf71..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/qt5.jam
+++ /dev/null
@@ -1,728 +0,0 @@
-# Copyright 2002-2006 Vladimir Prus
-# Copyright 2005 Alo Sarv
-# Copyright 2005-2012 Juergen Hunold
-#
-# Distributed under the Boost Software License, Version 1.0. (See
-# accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# Qt5 library support module
-#
-# The module attempts to auto-detect QT installation location from QTDIR
-# environment variable; failing that, installation location can be passed as
-# argument:
-#
-# toolset.using qt5 : /usr/local/Trolltech/Qt-5.0.0 ;
-#
-# The module supports code generation from .ui and .qrc files, as well as
-# running the moc preprocessor on headers. Note that you must list all your
-# moc-able headers in sources.
-#
-# Example:
-#
-# exe myapp : myapp.cpp myapp.h myapp.ui myapp.qrc
-# /qt5//QtGui /qt5//QtNetwork ;
-#
-# It's also possible to run moc on cpp sources:
-#
-# import cast ;
-#
-# exe myapp : myapp.cpp [ cast _ moccable-cpp : myapp.cpp ] /qt5//QtGui ;
-#
-# When moccing source file myapp.cpp you need to include "myapp.moc" from
-# myapp.cpp. When moccing .h files, the output of moc will be automatically
-# compiled and linked in, you don't need any includes.
-#
-# This is consistent with Qt guidelines:
-# http://qt-project.org/doc/qt-5.0/moc.html
-
-# The .qrc processing utility supports various command line option (see
-# http://qt-project.org/doc/qt-5.0/rcc.html for a complete list). The
-# module provides default arguments for the "output file" and
-# "initialization function name" options. Other options can be set through
-# the <rccflags> build property. E.g. if you wish the compression settings
-# to be more aggressive than the defaults, you can apply them too all .qrc
-# files like this:
-#
-# project my-qt-project :
-# requirements
-# <rccflags>"-compress 9 -threshold 10"
-# ;
-#
-# Of course, this property can also be specified on individual targets.
-
-
-import modules ;
-import feature ;
-import errors ;
-import type ;
-import "class" : new ;
-import generators ;
-import project ;
-import toolset : flags ;
-import os ;
-import virtual-target ;
-import scanner ;
-
-# The Qt version used for requirements
-# Valid are <qt>5.0 or <qt>5.1.0
-# Auto-detection via qmake sets '<qt>major.minor.patch'
-feature.feature qt5 : : propagated ;
-
-# Extra flags for rcc
-# $TODO: figure out how to declare this only once
-# feature.feature rccflags : : free ;
-
-project.initialize $(__name__) ;
-project qt5 ;
-
-# Save the project so that we tolerate 'import + using' combo.
-.project = [ project.current ] ;
-
-# Helper utils for easy debug output
-if [ MATCH (--debug-configuration) : [ modules.peek : ARGV ] ]
-{
- .debug-configuration = TRUE ;
-}
-
-local rule debug-message ( message * )
-{
- if $(.debug-configuration) = TRUE
- {
- ECHO notice: [qt5-cfg] $(message) ;
- }
-}
-
-# Capture qmake output line by line
-local rule read-output ( content )
-{
- local lines ;
- local nl = "
-" ;
- local << = "([^$(nl)]*)[$(nl)](.*)" ;
- local line+ = [ MATCH "$(<<)" : "$(content)" ] ;
- while $(line+)
- {
- lines += $(line+[1]) ;
- line+ = [ MATCH "$(<<)" : "$(line+[2])" ] ;
- }
- return $(lines) ;
-}
-
-# Capture Qt version from qmake
-local rule check-version ( bin_prefix )
-{
- full-cmd = $(bin_prefix)"/qmake -v" ;
- debug-message Running '$(full-cmd)' ;
- local output = [ SHELL $(full-cmd) ] ;
- for line in [ read-output $(output) ]
- {
- # Parse the output to get all the results.
- if [ MATCH "QMake" : $(line) ]
- {
- # Skip first line of output
- }
- else
- {
- temp = [ MATCH "([0-9]*)\\.([0-9]*)\\.([0-9]*)" : $(line) ] ;
- }
- }
- return $(temp) ;
-}
-
-# Validate the version string and extract the major/minor part we care about.
-#
-local rule split-version ( version )
-{
- local major-minor = [ MATCH ^([0-9]+)\.([0-9]+)(.*)$ : $(version) : 1 2 3 ] ;
- if ! $(major-minor[2]) || $(major-minor[3])
- {
- ECHO "Warning: 'using qt' expects a two part (major, minor) version number; got" $(version) instead ;
-
- # Add a zero to account for the missing digit if necessary.
- major-minor += 0 ;
- }
-
- return $(major-minor[1]) $(major-minor[2]) ;
-}
-
-# Initialize the QT support module.
-# Parameters:
-# - 'prefix' parameter tells where Qt is installed.
-# - 'version' optional version of Qt, else autodetected via 'qmake -v'
-# - 'condition' optional requirements
-# - 'namespace' optional support for configure -qtnamespace
-# - 'infix' optional support for configure -qtlibinfix
-# - 'full_bin' optional full path to Qt binaries (qmake,moc,uic,rcc)
-# - 'full_inc' optional full path to Qt top-level include directory
-# - 'full_lib' optional full path to Qt library directory
-rule init ( prefix : version ? : condition * : namespace ? : infix ? : full_bin ? : full_inc ? : full_lib ? )
-{
- project.push-current $(.project) ;
-
- debug-message "==== Configuring Qt ... ====" ;
- for local v in version prefix condition namespace infix full_bin full_inc full_lib
- {
- if $($(v))
- {
- debug-message " user-specified "$(v): '$($(v))' ;
- }
- }
-
- # Needed as default value
- .prefix = $(prefix) ;
-
- # pre-build paths to detect reinitializations changes
- local inc_prefix lib_prefix bin_prefix ;
- if $(full_inc)
- {
- inc_prefix = $(full_inc) ;
- }
- else
- {
- inc_prefix = $(prefix)/include ;
- }
- if $(full_lib)
- {
- lib_prefix = $(full_lib) ;
- }
- else
- {
- lib_prefix = $(prefix)/lib ;
- }
- if $(full_bin)
- {
- bin_prefix = $(full_bin) ;
- }
- else
- {
- bin_prefix = $(prefix)/bin ;
- }
-
- # Globally needed variables
- .incprefix = $(inc_prefix) ;
- .libprefix = $(lib_prefix) ;
- .binprefix = $(bin_prefix) ;
-
- if ! $(.initialized)
- {
- # Make sure this is initialised only once
- .initialized = true ;
-
- # Generates cpp files from header files using "moc" tool
- generators.register-standard qt5.moc : H : CPP(moc_%) : <allow>qt5 ;
-
- # The OBJ result type is a fake, 'H' will be really produced. See
- # comments on the generator class, defined below the 'init' function.
- generators.register [ new uic-5-generator qt5.uic : UI : OBJ :
- <allow>qt5 ] ;
-
- # The OBJ result type is a fake here too.
- generators.register [ new moc-h-5-generator
- qt5.moc.inc : MOCCABLE5_CPP : OBJ : <allow>qt5 ] ;
-
- generators.register [ new moc-inc-5-generator
- qt5.moc.inc : MOCCABLE5_H : OBJ : <allow>qt5 ] ;
-
- # Generates .cpp files from .qrc files.
- generators.register-standard qt5.rcc : QRC : CPP(qrc_%) : <allow>qt5 ;
-
- # dependency scanner for wrapped files.
- type.set-scanner QRC : qrc-5-scanner ;
-
- # Save value of first occuring prefix
- .PREFIX = $(prefix) ;
- }
-
- if $(version)
- {
- major-minor = [ split-version $(version) ] ;
- version = $(major-minor:J=.) ;
- }
- else
- {
- version = [ check-version $(bin_prefix) ] ;
- if $(version)
- {
- version = $(version:J=.) ;
- }
- debug-message Detected version '$(version)' ;
- }
-
- local target-requirements = $(condition) ;
-
- # Add the version, if any, to the target requirements.
- if $(version)
- {
- if ! $(version) in [ feature.values qt5 ]
- {
- feature.extend qt5 : $(version) ;
- }
- target-requirements += <qt5>$(version:E=default) ;
- }
-
- local target-os = [ feature.get-values target-os : $(condition) ] ;
- if ! $(target-os)
- {
- target-os ?= [ feature.defaults target-os ] ;
- target-os = $(target-os:G=) ;
- target-requirements += <target-os>$(target-os) ;
- }
-
- # Build exact requirements for the tools
- local tools-requirements = $(target-requirements:J=/) ;
-
- debug-message "Details of this Qt configuration:" ;
- debug-message " prefix: " '$(prefix:E=<empty>)' ;
- debug-message " binary path: " '$(bin_prefix:E=<empty>)' ;
- debug-message " include path:" '$(inc_prefix:E=<empty>)' ;
- debug-message " library path:" '$(lib_prefix:E=<empty>)' ;
- debug-message " target requirements:" '$(target-requirements)' ;
- debug-message " tool requirements: " '$(tools-requirements)' ;
-
- # setup the paths for the tools
- toolset.flags qt5.moc .BINPREFIX $(tools-requirements) : $(bin_prefix) ;
- toolset.flags qt5.rcc .BINPREFIX $(tools-requirements) : $(bin_prefix) ;
- toolset.flags qt5.uic .BINPREFIX $(tools-requirements) : $(bin_prefix) ;
-
- # TODO: 2009-02-12: Better support for directories
- # Most likely needed are separate getters for: include,libraries,binaries and sources.
- toolset.flags qt5.directory .PREFIX $(tools-requirements) : $(prefix) ;
-
- # Test for a buildable Qt.
- if [ glob $(.prefix)/Jamroot ]
- {
- .bjam-qt = true
-
- # this will declare QtCore (and qtmain on <target-os>windows)
- add-shared-library QtCore ;
- }
- else
- # Setup common pre-built Qt.
- # Special setup for QtCore on which everything depends
- {
- local link = [ feature.get-values link : $(condition) ] ;
-
- local usage-requirements =
- <include>$(.incprefix)
- <library-path>$(.libprefix)
- <threading>multi
- <allow>qt5 ;
-
- if $(link) in shared
- {
- usage-requirements += <dll-path>$(.libprefix) ;
- }
-
- local suffix ;
-
- # debug versions on unix have to be built
- # separately and therefore have no suffix.
- .infix_version = "" ;
- .suffix_debug = "" ;
-
- # Control flag for auto-configuration of the debug libraries.
- # This setup requires Qt 'configure -debug-and-release'.
- # Only available on some platforms.
- # ToDo: 2009-02-12: Maybe throw this away and
- # require separate setup with <variant>debug as condition.
- .have_separate_debug = FALSE ;
-
- # Setup other platforms
- if $(target-os) in windows cygwin
- {
- .have_separate_debug = TRUE ;
-
- # On NT, the libs have "d" suffix in debug builds.
- .suffix_debug = "d" ;
-
- .infix_version = "5" ;
-
- # On Windows we must link against the qtmain library
- lib qtmain
- : # sources
- : # requirements
- <name>qtmain$(.suffix_debug)
- <variant>debug
- $(target-requirements)
- ;
-
- lib qtmain
- : # sources
- : # requirements
- <name>qtmain
- $(target-requirements)
- ;
- }
- else if $(target-os) = darwin
- {
- # On MacOS X, both debug and release libraries are available.
- .suffix_debug = "_debug" ;
-
- .have_separate_debug = TRUE ;
-
- alias qtmain ;
- }
- else
- {
- alias qtmain : : $(target-requirements) ;
- .infix_version = "5" ;
- }
-
- lib QtCore : qtmain
- : # requirements
- <name>Qt$(.infix_version)Core
- $(target-requirements)
- : # default-build
- : # usage-requirements
- <define>QT_CORE_LIB
- <define>QT_NO_DEBUG
- <include>$(.incprefix)/QtCore
- $(usage-requirements)
- ;
-
- if $(.have_separate_debug) = TRUE
- {
- debug-message Configure debug libraries with suffix '$(.suffix_debug)' ;
-
- lib QtCore : $(main)
- : # requirements
- <name>Qt$(.infix_version)Core$(.suffix_debug)
- <variant>debug
- $(target-requirements)
- : # default-build
- : # usage-requirements
- <define>QT_CORE_LIB
- <include>$(.incprefix)/QtCore
- $(usage-requirements)
- ;
- }
- }
-
- # Initialising the remaining libraries is canonical
- # parameters 'module' : 'depends-on' : 'usage-define' : 'requirements' : 'include'
- # 'include' only for non-canonical include paths.
- add-shared-library QtGui : QtCore : QT_GUI_LIB : $(target-requirements) ;
- add-shared-library QtWidgets : QtGui : QT_WIDGETS_LIB : $(target-requirements) ;
- add-shared-library QtNetwork : QtCore : QT_NETWORK_LIB : $(target-requirements) ;
- add-shared-library QtSql : QtCore : QT_SQL_LIB : $(target-requirements) ;
- add-shared-library QtXml : QtCore : QT_XML_LIB : $(target-requirements) ;
- add-shared-library QtPrintSupport : QtGui : QT_PRINTSUPPORT_LIB : $(target-requirements) ;
- add-shared-library QtConcurrent : QtCore : QT_CONCURRENT_LIB : $(target-requirements) ;
-
- add-shared-library QtOpenGL : QtGui : QT_OPENGL_LIB : $(target-requirements) ;
- add-shared-library QtSvg : QtXml QtOpenGL : QT_SVG_LIB : $(target-requirements) ;
-
- add-shared-library QtTest : QtCore : : $(target-requirements) ;
-
- # Qt designer library et. al.
- add-shared-library QtDesigner : QtGui QtXml : : $(target-requirements) ;
- add-shared-library QtDesignerComponents : QtGui QtXml : : $(target-requirements) ;
- add-static-library QtUiTools : QtGui QtXml : $(target-requirements) ;
-
- # DBus-Support
- add-shared-library QtDBus : QtXml : : $(target-requirements) ;
-
- # Script-Engine and Tools
- add-shared-library QtScript : QtGui QtXml : QT_SCRIPT_LIB : $(target-requirements) ;
- add-shared-library QtScriptTools : QtScript : QT_SCRIPTTOOLS_LIB : $(target-requirements) ;
-
- # WebKit
- add-shared-library QtWebKit : QtGui : QT_WEBKIT_LIB : $(target-requirements) ;
- add-shared-library QtWebKitWidgets : QtGui : QT_WEBKITWIDGETS_LIB : $(target-requirements) ;
-
- # Multimedia engine
- add-shared-library QtMultimedia : QtGui : QT_MULTIMEDIA_LIB : $(target-requirements) ;
- add-shared-library QtMultimediaWidgets : QtMultimedia : QT_MULTIMEDIAWIDGETS_LIB : $(target-requirements) ;
-
- #
- add-shared-library QtXmlPatterns : QtNetwork : QT_XMLPATTERNS_LIB : $(target-requirements) ;
-
- # Help-Engine
- add-shared-library QtHelp : QtGui QtSql QtXml : : $(target-requirements) ;
- add-shared-library QtCLucene : QCore QtSql QtXml : : $(target-requirements) ;
-
- # QtQuick
- add-shared-library QtQml : QtCore QtNetwork QtGui : QT_QML_LIB : $(target-requirements) ;
- add-shared-library QtQuick : QtQml : QT_QUICK_LIB : $(target-requirements) ;
- add-shared-library QtQuickParticles : QtQml : : $(target-requirements) ;
- add-shared-library QtQuickTest : QtQml : : $(target-requirements) ;
-
- # Regular expression support
- add-shared-library QtV8 : QtCore : : $(target-requirements) ;
-
- # QML-Engine version1
- add-shared-library QtDeclarative : QtXml : : $(target-requirements) ;
-
- debug-message "==== Configured Qt-$(version) ====" ;
-
- project.pop-current ;
-}
-
-rule initialized ( )
-{
- return $(.initialized) ;
-}
-
-
-
-# This custom generator is needed because in QT5, UI files are translated only
-# into H files, and no C++ files are created. Further, the H files need not be
-# passed via MOC. The header is used only via inclusion. If we define a standard
-# UI -> H generator, Boost.Build will run MOC on H, and then compile the
-# resulting cpp. It will give a warning, since output from moc will be empty.
-#
-# This generator is declared with a UI -> OBJ signature, so it gets invoked when
-# linking generator tries to convert sources to OBJ, but it produces target of
-# type H. This is non-standard, but allowed. That header won't be mocced.
-#
-class uic-5-generator : generator
-{
- rule __init__ ( * : * )
- {
- generator.__init__ $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ;
- }
-
- rule run ( project name ? : property-set : sources * )
- {
- if ! $(name)
- {
- name = [ $(sources[0]).name ] ;
- name = $(name:B) ;
- }
-
- local a = [ new action $(sources[1]) : qt5.uic : $(property-set) ] ;
-
- # The 'ui_' prefix is to match qmake's default behavior.
- local target = [ new file-target ui_$(name) : H : $(project) : $(a) ] ;
-
- local r = [ virtual-target.register $(target) ] ;
-
- # Since this generator will return a H target, the linking generator
- # won't use it at all, and won't set any dependency on it. However, we
- # need the target to be seen by bjam, so that dependency from sources to
- # this generated header is detected -- if jam does not know about this
- # target, it won't do anything.
- DEPENDS all : [ $(r).actualize ] ;
-
- return $(r) ;
- }
-}
-
-
-class moc-h-5-generator : generator
-{
- rule __init__ ( * : * )
- {
- generator.__init__ $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ;
- }
-
- rule run ( project name ? : property-set : sources * )
- {
- if ! $(sources[2]) && [ $(sources[1]).type ] = MOCCABLE5_CPP
- {
- name = [ $(sources[0]).name ] ;
- name = $(name:B) ;
-
- local a = [ new action $(sources[1]) : qt5.moc.inc :
- $(property-set) ] ;
-
- local target = [ new file-target $(name) : MOC : $(project) : $(a)
- ] ;
-
- local r = [ virtual-target.register $(target) ] ;
-
- # Since this generator will return a H target, the linking generator
- # won't use it at all, and won't set any dependency on it. However,
- # we need the target to be seen by bjam, so that dependency from
- # sources to this generated header is detected -- if jam does not
- # know about this target, it won't do anything.
- DEPENDS all : [ $(r).actualize ] ;
-
- return $(r) ;
- }
- }
-}
-
-
-class moc-inc-5-generator : generator
-{
- rule __init__ ( * : * )
- {
- generator.__init__ $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ;
- }
-
- rule run ( project name ? : property-set : sources * )
- {
- if ! $(sources[2]) && [ $(sources[1]).type ] = MOCCABLE5_H
- {
- name = [ $(sources[0]).name ] ;
- name = $(name:B) ;
-
- local a = [ new action $(sources[1]) : qt5.moc.inc :
- $(property-set) ] ;
-
- local target = [ new file-target moc_$(name) : CPP : $(project) :
- $(a) ] ;
-
- # Since this generator will return a H target, the linking generator
- # won't use it at all, and won't set any dependency on it. However,
- # we need the target to be seen by bjam, so that dependency from
- # sources to this generated header is detected -- if jam does not
- # know about this target, it won't do anything.
- DEPENDS all : [ $(target).actualize ] ;
-
- return [ virtual-target.register $(target) ] ;
- }
- }
-}
-
-
-# Query the installation directory. This is needed in at least two scenarios.
-# First, when re-using sources from the Qt-Tree. Second, to "install" custom Qt
-# plugins to the Qt-Tree.
-#
-rule directory
-{
- return $(.PREFIX) ;
-}
-
-# Add a shared Qt library.
-rule add-shared-library ( lib-name : depends-on * : usage-defines * : requirements * : include ? )
-{
- add-library $(lib-name) : $(.infix_version) : $(depends-on) : $(usage-defines) : $(requirements) : $(include) ;
-}
-
-# Add a static Qt library.
-rule add-static-library ( lib-name : depends-on * : usage-defines * : requirements * : include ? )
-{
- add-library $(lib-name) : $(.infix_version) : $(depends-on) : $(usage-defines) : $(requirements) : $(include) ;
-}
-
-# Add a Qt library.
-# Static libs are unversioned, whereas shared libs have the major number as suffix.
-# Creates both release and debug versions on platforms where both are enabled by Qt configure.
-# Flags:
-# - lib-name Qt library Name
-# - version Qt major number used as shared library suffix (QtCore5.so)
-# - depends-on other Qt libraries
-# - usage-defines those are set by qmake, so set them when using this library
-# - requirements addional requirements
-# - include non-canonical include path. The canonical path is $(.incprefix)/$(lib-name).
-rule add-library ( lib-name : version ? : depends-on * : usage-defines * : requirements * : include ? )
-{
- if $(.bjam-qt)
- {
- # Import Qt module
- # Eveything will be setup there
- alias $(lib-name)
- : $(.prefix)//$(lib-name)
- :
- :
- : <allow>qt5 ;
- }
- else
- {
- local real_include ;
- real_include ?= $(include) ;
- real_include ?= $(lib-name) ;
-
- local real_name = [ MATCH ^Qt(.*) : $(lib-name) ] ;
-
- lib $(lib-name)
- : # sources
- $(depends-on)
- : # requirements
- <name>Qt$(version)$(real_name)
- $(requirements)
- : # default-build
- : # usage-requirements
- <define>$(usage-defines)
- <include>$(.incprefix)/$(real_include)
- ;
-
- if $(.have_separate_debug) = TRUE
- {
- lib $(lib-name)
- : # sources
- $(depends-on)
- : # requirements
- <name>Qt$(version)$(real_name)$(.suffix_debug)
- $(requirements)
- <variant>debug
- : # default-build
- : # usage-requirements
- <define>$(usage-defines)
- <include>$(.incprefix)/$(real_include)
- ;
- }
- }
-
- # Make library explicit so that a simple <use>qt5 will not bring in everything.
- # And some components like QtDBus/Phonon may not be available on all platforms.
- explicit $(lib-name) ;
-}
-
-# Use $(.BINPREFIX[-1]) for the paths as several tools-requirements can match.
-# The exact match is the last one.
-
-# Get <include> and <defines> from current toolset.
-flags qt5.moc INCLUDES <include> ;
-flags qt5.moc DEFINES <define> ;
-
-# need a newline for expansion of DEFINES and INCLUDES in the response file.
-.nl = "
-" ;
-
-# Processes headers to create Qt MetaObject information. Qt5-moc has its
-# c++-parser, so pass INCLUDES and DEFINES.
-# We use response file with one INCLUDE/DEFINE per line
-#
-actions moc
-{
- $(.BINPREFIX[-1])/moc -f $(>) -o $(<) @"@($(<).rsp:E=-D$(DEFINES)$(.nl) -I$(INCLUDES:T)$(.nl))"
-}
-
-# When moccing files for include only, we don't need -f, otherwise the generated
-# code will include the .cpp and we'll get duplicated symbols.
-#
-actions moc.inc
-{
- $(.BINPREFIX[-1])/moc $(>) -o $(<) @"@($(<).rsp:E=-D$(DEFINES)$(.nl) -I$(INCLUDES:T)$(.nl))"
-}
-
-
-# Get extra options for RCC
-flags qt5.rcc RCC_OPTIONS <rccflags> ;
-
-# Generates source files from resource files.
-#
-actions rcc
-{
- $(.BINPREFIX[-1])/rcc $(>) -name $(>:B) $(RCC_OPTIONS) -o $(<)
-}
-
-
-# Generates user-interface source from .ui files.
-#
-actions uic
-{
- $(.BINPREFIX[-1])/uic $(>) -o $(<)
-}
-
-
-# Scanner for .qrc files. Look for the CDATA section of the <file> tag. Ignore
-# the "alias" attribute. See http://doc.trolltech.com/qt/resources.html for
-# detailed documentation of the Qt Resource System.
-#
-class qrc-5-scanner : common-scanner
-{
- rule pattern ( )
- {
- return "<file.*>(.*)</file>" ;
- }
-}
-
-
-# Wrapped files are "included".
-scanner.register qrc-5-scanner : include ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/quickbook-config.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/quickbook-config.jam b/ext/kenlm/jam-files/boost-build/tools/quickbook-config.jam
deleted file mode 100644
index e983a78..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/quickbook-config.jam
+++ /dev/null
@@ -1,44 +0,0 @@
-#~ Copyright 2005 Rene Rivera.
-#~ Distributed under the Boost Software License, Version 1.0.
-#~ (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-# Automatic configuration for BoostBook tools. To use, just import this module.
-
-import os ;
-import toolset : using ;
-
-if [ os.name ] = NT
-{
- local boost-dir = ;
- for local R in snapshot cvs 1.33.0
- {
- boost-dir += [ W32_GETREG
- "HKEY_LOCAL_MACHINE\\SOFTWARE\\Boost.org\\$(R)"
- : "InstallRoot" ] ;
- }
- local quickbook-path = [ GLOB "$(boost-dir)\\bin" "\\Boost\\bin" : quickbook.exe ] ;
- quickbook-path = $(quickbook-path[1]) ;
-
- if $(quickbook-path)
- {
- if --debug-configuration in [ modules.peek : ARGV ]
- {
- ECHO "notice:" using quickbook ":" $(quickbook-path) ;
- }
- using quickbook : $(quickbook-path) ;
- }
-}
-else
-{
- local quickbook-path = [ GLOB "/usr/local/bin" "/usr/bin" "/opt/bin" : quickbook ] ;
- quickbook-path = $(quickbook-path[1]) ;
-
- if $(quickbook-path)
- {
- if --debug-configuration in [ modules.peek : ARGV ]
- {
- ECHO "notice:" using quickbook ":" $(quickbook-path) ;
- }
- using quickbook : $(quickbook-path) ;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/quickbook.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/quickbook.jam b/ext/kenlm/jam-files/boost-build/tools/quickbook.jam
deleted file mode 100644
index 6de2d42..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/quickbook.jam
+++ /dev/null
@@ -1,361 +0,0 @@
-#
-# Copyright (c) 2005 Jo�o Abecasis
-# Copyright (c) 2005 Vladimir Prus
-# Copyright (c) 2006 Rene Rivera
-#
-# Distributed under the Boost Software License, Version 1.0. (See
-# accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-#
-
-# This toolset defines a generator to translate QuickBook to BoostBook. It can
-# be used to generate nice (!) user documentation in different formats
-# (pdf/html/...), from a single text file with simple markup.
-#
-# The toolset defines the QUICKBOOK type (file extension 'qbk') and
-# a QUICKBOOK to XML (BOOSTBOOK) generator.
-#
-#
-# ===========================================================================
-# Q & A
-# ===========================================================================
-#
-# If you don't know what this is all about, some Q & A will hopefully get you
-# up to speed with QuickBook and this toolset.
-#
-#
-# What is QuickBook ?
-#
-# QuickBook is a WikiWiki style documentation tool geared towards C++
-# documentation using simple rules and markup for simple formatting tasks.
-# QuickBook extends the WikiWiki concept. Like the WikiWiki, QuickBook
-# documents are simple text files. A single QuickBook document can
-# generate a fully linked set of nice HTML and PostScript/PDF documents
-# complete with images and syntax-colorized source code.
-#
-#
-# Where can I get QuickBook ?
-#
-# Quickbook can be found in Boost's repository, under the tools/quickbook
-# directory it was added there on Jan 2005, some time after the release of
-# Boost v1.32.0 and has been an integral part of the Boost distribution
-# since v1.33.
-#
-# Here's a link to the SVN repository:
-# https://svn.boost.org/svn/boost/trunk/tools/quickbook
-#
-# And to QuickBook's QuickBook-generated docs:
-# http://www.boost.org/doc/libs/release/tools/quickbook/index.html
-#
-#
-# How do I use QuickBook and this toolset in my projects ?
-#
-# The minimal example is:
-#
-# using boostbook ;
-# import quickbook ;
-#
-# boostbook my_docs : my_docs_source.qbk ;
-#
-# where my_docs is a target name and my_docs_source.qbk is a QuickBook
-# file. The documentation format to be generated is determined by the
-# boostbook toolset. By default html documentation should be generated,
-# but you should check BoostBook's docs to be sure.
-#
-#
-# What do I need ?
-#
-# You should start by setting up the BoostBook toolset. Please refer to
-# boostbook.jam and the BoostBook documentation for information on how to
-# do this.
-#
-# A QuickBook executable is also needed. The toolset will generate this
-# executable if it can find the QuickBook sources. The following
-# directories will be searched:
-#
-# BOOST_ROOT/tools/quickbook/
-# BOOST_BUILD_PATH/../../quickbook/
-#
-# (BOOST_ROOT and BOOST_BUILD_PATH are environment variables)
-#
-# If QuickBook sources are not found the toolset will then try to use
-# the shell command 'quickbook'.
-#
-#
-# How do I provide a custom QuickBook executable ?
-#
-# You may put the following in your user-config.jam or site-config.jam:
-#
-# using quickbook : /path/to/quickbook ;
-#
-# or, if 'quickbook' can be found in your PATH,
-#
-# using quickbook : quickbook ;
-#
-#
-# For convenience three alternatives are tried to get a QuickBook executable:
-#
-# 1. If the user points us to the a QuickBook executable, that is used.
-#
-# 2. Otherwise, we search for the QuickBook sources and compile QuickBook
-# using the default toolset.
-#
-# 3. As a last resort, we rely on the shell for finding 'quickbook'.
-#
-
-import boostbook ;
-import "class" : new ;
-import feature ;
-import generators ;
-import toolset ;
-import type ;
-import scanner ;
-import project ;
-import targets ;
-import build-system ;
-import path ;
-import common ;
-import errors ;
-
-# The one and only QUICKBOOK type!
-type.register QUICKBOOK : qbk ;
-
-# <quickbook-binary> shell command to run QuickBook
-# <quickbook-binary-dependencies> targets to build QuickBook from sources.
-feature.feature <quickbook-binary> : : free ;
-feature.feature <quickbook-binary-dependencies> : : free dependency ;
-feature.feature <quickbook-define> : : free ;
-feature.feature <quickbook-indent> : : free ;
-feature.feature <quickbook-line-width> : : free ;
-
-
-# quickbook-binary-generator handles generation of the QuickBook executable, by
-# marking it as a dependency for QuickBook docs.
-#
-# If the user supplied the QuickBook command that will be used.
-#
-# Otherwise we search some sensible places for the QuickBook sources and compile
-# from scratch using the default toolset.
-#
-# As a last resort we rely on the shell to find 'quickbook'.
-#
-class quickbook-binary-generator : generator
-{
- import modules path targets quickbook ;
-
- rule run ( project name ? : property-set : sources * : multiple ? )
- {
- quickbook.freeze-config ;
- # QuickBook invocation command and dependencies.
- local quickbook-binary = [ modules.peek quickbook : .quickbook-binary ] ;
- local quickbook-binary-dependencies ;
-
- if ! $(quickbook-binary)
- {
- # If the QuickBook source directory was found, mark its main target
- # as a dependency for the current project. Otherwise, try to find
- # 'quickbook' in user's PATH
- local quickbook-dir = [ modules.peek quickbook : .quickbook-dir ] ;
- if $(quickbook-dir)
- {
- # Get the main-target in QuickBook directory.
- local quickbook-main-target = [ targets.resolve-reference $(quickbook-dir) : $(project) ] ;
-
- # The first element are actual targets, the second are
- # properties found in target-id. We do not care about these
- # since we have passed the id ourselves.
- quickbook-main-target =
- [ $(quickbook-main-target[1]).main-target quickbook ] ;
-
- quickbook-binary-dependencies =
- [ $(quickbook-main-target).generate [ $(property-set).propagated ] ] ;
-
- # Ignore usage-requirements returned as first element.
- quickbook-binary-dependencies = $(quickbook-binary-dependencies[2-]) ;
-
- # Some toolsets generate extra targets (e.g. RSP). We must mark
- # all targets as dependencies for the project, but we will only
- # use the EXE target for quickbook-to-boostbook translation.
- for local target in $(quickbook-binary-dependencies)
- {
- if [ $(target).type ] = EXE
- {
- quickbook-binary =
- [ path.native
- [ path.join
- [ $(target).path ]
- [ $(target).name ]
- ]
- ] ;
- }
- }
- }
- }
-
- # Add $(quickbook-binary-dependencies) as a dependency of the current
- # project and set it as the <quickbook-binary> feature for the
- # quickbook-to-boostbook rule, below.
- property-set = [ $(property-set).add-raw
- <dependency>$(quickbook-binary-dependencies)
- <quickbook-binary>$(quickbook-binary)
- <quickbook-binary-dependencies>$(quickbook-binary-dependencies)
- ] ;
-
- return [ generator.run $(project) $(name) : $(property-set) : $(sources) : $(multiple) ] ;
- }
-}
-
-
-# Define a scanner for tracking QBK include dependencies.
-#
-class qbk-scanner : common-scanner
-{
- rule pattern ( )
- {
- return "\\[[ ]*include[ ]+([^]]+)\\]"
- "\\[[ ]*include:[a-zA-Z0-9_]+[ ]+([^]]+)\\]"
- "\\[[ ]*import[ ]+([^]]+)\\]" ;
- }
-}
-
-
-scanner.register qbk-scanner : include ;
-
-type.set-scanner QUICKBOOK : qbk-scanner ;
-
-
-# Initialization of toolset.
-#
-# Parameters:
-# command ? -> path to QuickBook executable.
-#
-# When command is not supplied toolset will search for QuickBook directory and
-# compile the executable from source. If that fails we still search the path for
-# 'quickbook'.
-#
-rule init (
- command ? # path to the QuickBook executable.
- )
-{
- if $(command)
- {
- if $(.config-frozen)
- {
- errors.user-error "quickbook: configuration cannot be changed after it has been used." ;
- }
- .command = $(command) ;
- }
-}
-
-rule freeze-config ( )
-{
- if ! $(.config-frozen)
- {
- .config-frozen = true ;
-
- # QuickBook invocation command and dependencies.
-
- .quickbook-binary = $(.command) ;
-
- if $(.quickbook-binary)
- {
- # Use user-supplied command.
- .quickbook-binary = [ common.get-invocation-command quickbook : quickbook : $(.quickbook-binary) ] ;
- }
- else
- {
- # Search for QuickBook sources in sensible places, like
- # $(BOOST_ROOT)/tools/quickbook
- # $(BOOST_BUILD_PATH)/../../quickbook
-
- # And build quickbook executable from sources.
-
- local boost-root = [ modules.peek : BOOST_ROOT ] ;
- local boost-build-path = [ build-system.location ] ;
-
- if $(boost-root)
- {
- .quickbook-dir += [ path.join $(boost-root) tools ] ;
- }
-
- if $(boost-build-path)
- {
- .quickbook-dir += $(boost-build-path)/../.. ;
- }
-
- .quickbook-dir = [ path.glob $(.quickbook-dir) : quickbook ] ;
-
- # If the QuickBook source directory was found, mark its main target
- # as a dependency for the current project. Otherwise, try to find
- # 'quickbook' in user's PATH
- if $(.quickbook-dir)
- {
- .quickbook-dir = [ path.make $(.quickbook-dir[1]) ] ;
- }
- else
- {
- ECHO "QuickBook warning: The path to the quickbook executable was" ;
- ECHO " not provided. Additionally, couldn't find QuickBook" ;
- ECHO " sources searching in" ;
- ECHO " * BOOST_ROOT/tools/quickbook" ;
- ECHO " * BOOST_BUILD_PATH/../../quickbook" ;
- ECHO " Will now try to find a precompiled executable by searching" ;
- ECHO " the PATH for 'quickbook'." ;
- ECHO " To disable this warning in the future, or to completely" ;
- ECHO " avoid compilation of quickbook, you can explicitly set the" ;
- ECHO " path to a quickbook executable command in user-config.jam" ;
- ECHO " or site-config.jam with the call" ;
- ECHO " using quickbook : /path/to/quickbook ;" ;
-
- # As a last resort, search for 'quickbook' command in path. Note
- # that even if the 'quickbook' command is not found,
- # get-invocation-command will still return 'quickbook' and might
- # generate an error while generating the virtual-target.
-
- .quickbook-binary = [ common.get-invocation-command quickbook : quickbook ] ;
- }
- }
- }
-}
-
-
-generators.register [ new quickbook-binary-generator quickbook.quickbook-to-boostbook : QUICKBOOK : XML ] ;
-
-
-# <quickbook-binary> shell command to run QuickBook
-# <quickbook-binary-dependencies> targets to build QuickBook from sources.
-toolset.flags quickbook.quickbook-to-boostbook QB-COMMAND <quickbook-binary> ;
-toolset.flags quickbook.quickbook-to-boostbook QB-DEPENDENCIES <quickbook-binary-dependencies> ;
-toolset.flags quickbook.quickbook-to-boostbook INCLUDES <include> ;
-toolset.flags quickbook.quickbook-to-boostbook QB-DEFINES <quickbook-define> ;
-toolset.flags quickbook.quickbook-to-boostbook QB-INDENT <quickbook-indent> ;
-toolset.flags quickbook.quickbook-to-boostbook QB-LINE-WIDTH <quickbook-line-width> ;
-
-
-rule quickbook-to-boostbook ( target : source : properties * )
-{
- # Signal dependency of quickbook sources on <quickbook-binary-dependencies>
- # upon invocation of quickbook-to-boostbook.
- DEPENDS $(target) : [ on $(target) return $(QB-DEPENDENCIES) ] ;
-}
-
-
-actions quickbook-to-boostbook
-{
- "$(QB-COMMAND)" -I"$(INCLUDES)" -D"$(QB-DEFINES)" --indent="$(QB-INDENT)" --linewidth="$(QB-LINE-WIDTH)" --output-file="$(1)" "$(2)"
-}
-
-
-# Declare a main target to convert a quickbook source into a boostbook XML file.
-#
-rule to-boostbook ( target-name : sources * : requirements * : default-build * )
-{
- local project = [ project.current ] ;
-
- targets.main-target-alternative
- [ new typed-target $(target-name) : $(project) : XML
- : [ targets.main-target-sources $(sources) : $(target-name) ]
- : [ targets.main-target-requirements $(requirements) : $(project) ]
- : [ targets.main-target-default-build $(default-build) : $(project) ]
- ] ;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/rc.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/rc.jam b/ext/kenlm/jam-files/boost-build/tools/rc.jam
deleted file mode 100644
index de4071f..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/rc.jam
+++ /dev/null
@@ -1,155 +0,0 @@
-# Copyright (C) Andre Hentz 2003. Permission to copy, use, modify, sell and
-# distribute this software is granted provided this copyright notice appears in
-# all copies. This software is provided "as is" without express or implied
-# warranty, and with no claim as to its suitability for any purpose.
-#
-# Copyright (c) 2006 Rene Rivera.
-#
-# Use, modification and distribution is subject to the Boost Software
-# License Version 1.0. (See accompanying file LICENSE_1_0.txt or
-# http://www.boost.org/LICENSE_1_0.txt)
-
-import generators ;
-import feature ;
-import scanner ;
-import toolset : flags ;
-import type ;
-
-if [ MATCH (--debug-configuration) : [ modules.peek : ARGV ] ]
-{
- .debug-configuration = true ;
-}
-
-type.register RC : rc ;
-
-rule init ( )
-{
-}
-
-# Configures a new resource compilation command specific to a condition,
-# usually a toolset selection condition. The possible options are:
-#
-# * <rc-type>(rc|windres) - Indicates the type of options the command
-# accepts.
-#
-# Even though the arguments are all optional, only when a command, condition,
-# and at minimum the rc-type option are given will the command be configured.
-# This is so that callers don't have to check auto-configuration values before
-# calling this. And still get the functionality of build failures when the
-# resource compiler can not be found.
-#
-rule configure ( command ? : condition ? : options * )
-{
- local rc-type = [ feature.get-values <rc-type> : $(options) ] ;
-
- if $(command) && $(condition) && $(rc-type)
- {
- flags rc.compile.resource .RC $(condition) : $(command) ;
- flags rc.compile.resource .RC_TYPE $(condition) : $(rc-type:L) ;
- flags rc.compile.resource DEFINES <define> ;
- flags rc.compile.resource INCLUDES <include> ;
- if $(.debug-configuration)
- {
- ECHO notice: using rc compiler :: $(condition) :: $(command) ;
- }
- }
-}
-
-rule compile.resource ( target : sources * : properties * )
-{
- local rc-type = [ on $(target) return $(.RC_TYPE) ] ;
- rc-type ?= null ;
- compile.resource.$(rc-type) $(target) : $(sources[1]) ;
-}
-
-actions compile.resource.rc
-{
- "$(.RC)" -l 0x409 "-U$(UNDEFS)" "-D$(DEFINES)" -I"$(>:D)" -I"$(<:D)" -I"$(INCLUDES)" -fo "$(<)" "$(>)"
-}
-
-actions compile.resource.windres
-{
- "$(.RC)" "-U$(UNDEFS)" "-D$(DEFINES)" -I"$(>:D)" -I"$(<:D)" -I"$(INCLUDES)" -o "$(<)" -i "$(>)"
-}
-
-actions quietly compile.resource.null
-{
- as /dev/null -o "$(<)"
-}
-
-# Since it is common practice to write
-# exe hello : hello.cpp hello.rc
-# we change the name of object created from RC file, to avoid conflict with
-# hello.cpp. The reason we generate OBJ and not RES, is that gcc does not seem
-# to like RES files, but works OK with OBJ (see
-# http://article.gmane.org/gmane.comp.lib.boost.build/5643).
-#
-# Using 'register-c-compiler' adds the build directory to INCLUDES
-generators.register-c-compiler rc.compile.resource : RC : OBJ(%_res) ;
-
-# Register scanner for resources
-class res-scanner : scanner
-{
- import path ;
- import regex ;
- import scanner ;
- import virtual-target ;
-
- rule __init__ ( includes * )
- {
- scanner.__init__ ;
- self.includes = $(includes) ;
- }
-
- rule pattern ( )
- {
- return "(([^ ]+[ ]+(BITMAP|CURSOR|FONT|ICON|MESSAGETABLE|RT_MANIFEST)[ ]+([^ \"]+|\"[^\"]+\"))|(#include[ ]*(<[^<]+>|\"[^\"]+\")))" ;
- }
-
- rule process ( target : matches * : binding )
- {
- local angle = [ regex.transform $(matches) : "#include[ ]*<([^<]+)>" ] ;
- local quoted = [ regex.transform $(matches) : "#include[ ]*\"([^\"]+)\"" ] ;
- local res = [ regex.transform $(matches) : "[^ ]+[ ]+(BITMAP|CURSOR|FONT|ICON|MESSAGETABLE|RT_MANIFEST)[ ]+(([^ \"]+)|\"([^\"]+)\")" : 3 4 ] ;
-
- # Icons and other includes may be referenced as
- #
- # IDR_MAINFRAME ICON "res\\icon.ico"
- #
- # so we have to replace double backslashes with single ones.
- res = [ regex.replace-list $(res) : "\\\\\\\\" : "/" ] ;
-
- # CONSIDER: the new scoping rules seem to defeat "on target" variables.
- local g = [ on $(target) return $(HDRGRIST) ] ;
- local b = [ NORMALIZE_PATH $(binding:D) ] ;
-
- # Attach binding of including file to included targets. When a target is
- # directly created from a virtual target this extra information is
- # unnecessary. But in other cases, it allows us to distinguish between
- # two headers of the same name included from different places. We do not
- # need this extra information for angle includes, since they should not
- # depend on the including file (we can not get literal "." in the
- # include path).
- local g2 = $(g)"#"$(b) ;
-
- angle = $(angle:G=$(g)) ;
- quoted = $(quoted:G=$(g2)) ;
- res = $(res:G=$(g2)) ;
-
- local all = $(angle) $(quoted) $(res) ;
-
- INCLUDES $(target) : $(all) ;
- NOCARE $(all) ;
- SEARCH on $(angle) = $(self.includes:G=) ;
- SEARCH on $(quoted) $(res) = $(b) $(self.includes:G=) ;
-
- # Just propagate the current scanner to includes, in hope that includes
- # do not change scanners.
- scanner.propagate $(__name__) : $(angle) $(quoted) : $(target) ;
-
- ISFILE $(all) ;
- }
-}
-
-scanner.register res-scanner : include ;
-type.set-scanner RC : res-scanner ;
[38/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/intel-linux.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/intel-linux.jam b/ext/kenlm/jam-files/boost-build/tools/intel-linux.jam
deleted file mode 100644
index d9164ad..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/intel-linux.jam
+++ /dev/null
@@ -1,250 +0,0 @@
-# Copyright (c) 2003 Michael Stevens
-# Copyright (c) 2011 Bryce Lelbach
-#
-# Use, modification and distribution is subject to the Boost Software
-# License Version 1.0. (See accompanying file LICENSE_1_0.txt or
-# http://www.boost.org/LICENSE_1_0.txt)
-
-import toolset ;
-import feature ;
-import toolset : flags ;
-
-import intel ;
-import gcc ;
-import common ;
-import errors ;
-import generators ;
-import type ;
-import numbers ;
-
-feature.extend-subfeature toolset intel : platform : linux ;
-
-toolset.inherit-generators intel-linux
- <toolset>intel <toolset-intel:platform>linux : gcc : gcc.mingw.link gcc.mingw.link.dll ;
-generators.override intel-linux.prebuilt : builtin.lib-generator ;
-generators.override intel-linux.prebuilt : builtin.prebuilt ;
-generators.override intel-linux.searched-lib-generator : searched-lib-generator ;
-
-# Override default do-nothing generators.
-generators.override intel-linux.compile.c.pch : pch.default-c-pch-generator ;
-generators.override intel-linux.compile.c++.pch : pch.default-cpp-pch-generator ;
-
-type.set-generated-target-suffix PCH : <toolset>intel <toolset-intel:platform>linux : pchi ;
-
-toolset.inherit-rules intel-linux : gcc ;
-toolset.inherit-flags intel-linux : gcc
- : <inlining>off <inlining>on <inlining>full
- <optimization>space <optimization>speed
- <warnings>off <warnings>all <warnings>on
- ;
-
-if [ MATCH (--debug-configuration) : [ modules.peek : ARGV ] ]
-{
- .debug-configuration = true ;
-}
-
-# Initializes the intel-linux toolset
-# version in mandatory
-# name (default icpc) is used to invoke the specified intel-linux complier
-# compile and link options allow you to specify addition command line options for each version
-rule init ( version ? : command * : options * )
-{
- local condition = [ common.check-init-parameters intel-linux
- : version $(version) ] ;
-
- if $(.debug-configuration)
- {
- ECHO "notice: intel-linux version is" $(version) ;
- }
-
- local default_path ;
-
- # Intel C++ Composer XE 2011 for Linux, aka Intel C++ Compiler XE 12.0,
- # aka intel-linux-12.0. In this version, Intel thankfully decides to install
- # to a sane 'intel' folder in /opt.
- if [ MATCH "(12[.]0|12)" : $(version) ]
- { default_path = /opt/intel/bin ; }
- # Intel C++ Compiler 11.1.
- else if [ MATCH "(11[.]1)" : $(version) ]
- { default_path = /opt/intel_cce_11.1.064.x86_64/bin ; }
- # Intel C++ Compiler 11.0.
- else if [ MATCH "(11[.]0|11)" : $(version) ]
- { default_path = /opt/intel_cce_11.0.074.x86_64/bin ; }
- # Intel C++ Compiler 10.1.
- else if [ MATCH "(10[.]1)" : $(version) ]
- { default_path = /opt/intel_cce_10.1.013_x64/bin ; }
- # Intel C++ Compiler 9.1.
- else if [ MATCH "(9[.]1)" : $(version) ]
- { default_path = /opt/intel_cc_91/bin ; }
- # Intel C++ Compiler 9.0.
- else if [ MATCH "(9[.]0|9)" : $(version) ]
- { default_path = /opt/intel_cc_90/bin ; }
- # Intel C++ Compiler 8.1.
- else if [ MATCH "(8[.]1)" : $(version) ]
- { default_path = /opt/intel_cc_81/bin ; }
- # Intel C++ Compiler 8.0 - this used to be the default, so now it's the
- # fallback.
- else
- { default_path = /opt/intel_cc_80/bin ; }
-
- if $(.debug-configuration)
- {
- ECHO "notice: default search path for intel-linux is" $(default_path) ;
- }
-
- command = [ common.get-invocation-command intel-linux : icpc
- : $(command) : $(default_path) ] ;
-
- common.handle-options intel-linux : $(condition) : $(command) : $(options) ;
-
- gcc.init-link-flags intel-linux gnu $(condition) ;
-
- local root = [ feature.get-values <root> : $(options) ] ;
- local bin ;
- if $(command) || $(root)
- {
- bin ?= [ common.get-absolute-tool-path $(command[-1]) ] ;
- root ?= $(bin:D) ;
-
- local command-string = $(command:J=" ") ;
- local version-output = [ SHELL "$(command-string) --version" ] ;
- local real-version = [ MATCH "([0-9.]+)" : $(version-output) ] ;
- local major = [ MATCH "([0-9]+).*" : $(real-version) ] ;
-
- # If we failed to determine major version, use the behaviour for
- # the current compiler.
- if $(major) && [ numbers.less $(major) 10 ]
- {
- flags intel-linux.compile OPTIONS $(condition)/<inlining>off : "-Ob0" ;
- flags intel-linux.compile OPTIONS $(condition)/<inlining>on : "-Ob1" ;
- flags intel-linux.compile OPTIONS $(condition)/<inlining>full : "-Ob2" ;
- flags intel-linux.compile OPTIONS $(condition)/<optimization>space : "-O1" ;
- flags intel-linux.compile OPTIONS $(condition)/<optimization>speed : "-O3 -ip" ;
- }
- else if $(major) && [ numbers.less $(major) 11 ]
- {
- flags intel-linux.compile OPTIONS $(condition)/<inlining>off : "-inline-level=0" ;
- flags intel-linux.compile OPTIONS $(condition)/<inlining>on : "-inline-level=1" ;
- flags intel-linux.compile OPTIONS $(condition)/<inlining>full : "-inline-level=2" ;
- flags intel-linux.compile OPTIONS $(condition)/<optimization>space : "-O1" ;
- flags intel-linux.compile OPTIONS $(condition)/<optimization>speed : "-O3 -ip" ;
- }
- else # newer version of intel do have -Os (at least 11+, don't know about 10)
- {
- flags intel-linux.compile OPTIONS $(condition)/<inlining>off : "-inline-level=0" ;
- flags intel-linux.compile OPTIONS $(condition)/<inlining>on : "-inline-level=1" ;
- flags intel-linux.compile OPTIONS $(condition)/<inlining>full : "-inline-level=2" ;
- flags intel-linux.compile OPTIONS $(condition)/<optimization>space : "-Os" ;
- flags intel-linux.compile OPTIONS $(condition)/<optimization>speed : "-O3 -ip" ;
- }
-
- if $(root)
- {
- # Libraries required to run the executable may be in either
- # $(root)/lib (10.1 and earlier)
- # or
- # $(root)/lib/architecture-name (11.0 and later:
- local lib_path = $(root)/lib $(root:P)/lib/$(bin:B) ;
- if $(.debug-configuration)
- {
- ECHO notice: using intel libraries :: $(condition) :: $(lib_path) ;
- }
- flags intel-linux.link RUN_PATH $(condition) : $(lib_path) ;
- }
- }
-}
-
-SPACE = " " ;
-
-flags intel-linux.compile OPTIONS <warnings>off : -w0 ;
-flags intel-linux.compile OPTIONS <warnings>on : -w1 ;
-flags intel-linux.compile OPTIONS <warnings>all : -w2 ;
-
-rule compile.c++ ( targets * : sources * : properties * )
-{
- gcc.setup-threading $(targets) : $(sources) : $(properties) ;
- gcc.setup-fpic $(targets) : $(sources) : $(properties) ;
- gcc.setup-address-model $(targets) : $(sources) : $(properties) ;
- DEPENDS $(<) : [ on $(<) return $(PCH_FILE) ] ;
-}
-
-actions compile.c++ bind PCH_FILE
-{
- "$(CONFIG_COMMAND)" -c -xc++ $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -use-pch"$(PCH_FILE)" -c -o "$(<)" "$(>)"
-}
-
-rule compile.c ( targets * : sources * : properties * )
-{
- gcc.setup-threading $(targets) : $(sources) : $(properties) ;
- gcc.setup-fpic $(targets) : $(sources) : $(properties) ;
- gcc.setup-address-model $(targets) : $(sources) : $(properties) ;
- DEPENDS $(<) : [ on $(<) return $(PCH_FILE) ] ;
-}
-
-actions compile.c bind PCH_FILE
-{
- "$(CONFIG_COMMAND)" -c -xc $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -use-pch"$(PCH_FILE)" -c -o "$(<)" "$(>)"
-}
-
-rule compile.c++.pch ( targets * : sources * : properties * )
-{
- gcc.setup-threading $(targets) : $(sources) : $(properties) ;
- gcc.setup-fpic $(targets) : $(sources) : $(properties) ;
- gcc.setup-address-model $(targets) : $(sources) : $(properties) ;
-}
-#
-# Compiling a pch first deletes any existing *.pchi file, as Intel's compiler
-# won't over-write an existing pch: instead it creates filename$1.pchi, filename$2.pchi
-# etc - which appear not to do anything except take up disk space :-(
-#
-actions compile.c++.pch
-{
- rm -f "$(<)" && "$(CONFIG_COMMAND)" -x c++-header $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -pch-create "$(<)" "$(>)"
-}
-
-actions compile.fortran
-{
- "ifort" -c $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
-}
-
-rule compile.c.pch ( targets * : sources * : properties * )
-{
- gcc.setup-threading $(targets) : $(sources) : $(properties) ;
- gcc.setup-fpic $(targets) : $(sources) : $(properties) ;
- gcc.setup-address-model $(targets) : $(sources) : $(properties) ;
-}
-
-actions compile.c.pch
-{
- rm -f "$(<)" && "$(CONFIG_COMMAND)" -x c-header $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -pch-create "$(<)" "$(>)"
-}
-
-rule link ( targets * : sources * : properties * )
-{
- gcc.setup-threading $(targets) : $(sources) : $(properties) ;
- gcc.setup-address-model $(targets) : $(sources) : $(properties) ;
- SPACE on $(targets) = " " ;
- JAM_SEMAPHORE on $(targets) = <s>intel-linux-link-semaphore ;
-}
-
-actions link bind LIBRARIES
-{
- "$(CONFIG_COMMAND)" -L"$(LINKPATH)" -Wl,-R$(SPACE)-Wl,"$(RPATH)" -Wl,-rpath-link$(SPACE)-Wl,"$(RPATH_LINK)" -o "$(<)" "$(>)" "$(LIBRARIES)" -l$(FINDLIBS-SA) -l$(FINDLIBS-ST) $(OPTIONS) $(USER_OPTIONS)
-}
-
-rule link.dll ( targets * : sources * : properties * )
-{
- gcc.setup-threading $(targets) : $(sources) : $(properties) ;
- gcc.setup-address-model $(targets) : $(sources) : $(properties) ;
- SPACE on $(targets) = " " ;
- JAM_SEMAPHORE on $(targets) = <s>intel-linux-link-semaphore ;
-}
-
-# Differ from 'link' above only by -shared.
-actions link.dll bind LIBRARIES
-{
- "$(CONFIG_COMMAND)" -L"$(LINKPATH)" -Wl,-R$(SPACE)-Wl,"$(RPATH)" -o "$(<)" -Wl,-soname$(SPACE)-Wl,$(<[1]:D=) -shared "$(>)" "$(LIBRARIES)" -l$(FINDLIBS-SA) -l$(FINDLIBS-ST) $(OPTIONS) $(USER_OPTIONS)
-}
-
-
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/intel-win.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/intel-win.jam b/ext/kenlm/jam-files/boost-build/tools/intel-win.jam
deleted file mode 100644
index c9adac0..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/intel-win.jam
+++ /dev/null
@@ -1,184 +0,0 @@
-# Copyright Vladimir Prus 2004.
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt
-# or copy at http://www.boost.org/LICENSE_1_0.txt)
-
-# Importing common is needed because the rules we inherit here depend on it.
-# That is nasty.
-import common ;
-import errors ;
-import feature ;
-import intel ;
-import msvc ;
-import os ;
-import toolset ;
-import generators ;
-import type ;
-
-feature.extend-subfeature toolset intel : platform : win ;
-
-toolset.inherit-generators intel-win <toolset>intel <toolset-intel:platform>win : msvc ;
-toolset.inherit-flags intel-win : msvc : : YLOPTION ;
-toolset.inherit-rules intel-win : msvc ;
-
-# Override default do-nothing generators.
-generators.override intel-win.compile.c.pch : pch.default-c-pch-generator ;
-generators.override intel-win.compile.c++.pch : pch.default-cpp-pch-generator ;
-generators.override intel-win.compile.rc : rc.compile.resource ;
-generators.override intel-win.compile.mc : mc.compile ;
-
-toolset.flags intel-win.compile PCH_SOURCE <pch>on : <pch-source> ;
-
-toolset.add-requirements <toolset>intel-win,<runtime-link>shared:<threading>multi ;
-
-# Initializes the intel toolset for windows
-rule init ( version ? : # the compiler version
- command * : # the command to invoke the compiler itself
- options * # Additional option: <compatibility>
- # either 'vc6', 'vc7', 'vc7.1'
- # or 'native'(default).
- )
-{
- local compatibility =
- [ feature.get-values <compatibility> : $(options) ] ;
- local condition = [ common.check-init-parameters intel-win
- : version $(version) : compatibility $(compatibility) ] ;
-
- command = [ common.get-invocation-command intel-win : icl.exe :
- $(command) ] ;
-
- common.handle-options intel-win : $(condition) : $(command) : $(options) ;
-
- local root ;
- if $(command)
- {
- root = [ common.get-absolute-tool-path $(command[-1]) ] ;
- root = $(root)/ ;
- }
-
- local setup ;
- setup = [ GLOB $(root) : iclvars_*.bat ] ;
- if ! $(setup)
- {
- setup = $(root)/iclvars.bat ;
- }
- setup = "call \""$(setup)"\" > nul " ;
-
- if [ os.name ] = NT
- {
- setup = $(setup)"
-" ;
- }
- else
- {
- setup = "cmd /S /C "$(setup)" \"&&\" " ;
- }
-
- toolset.flags intel-win.compile .CC $(condition) : $(setup)icl ;
- toolset.flags intel-win.link .LD $(condition) : $(setup)xilink ;
- toolset.flags intel-win.archive .LD $(condition) : $(setup)xilink /lib ;
- toolset.flags intel-win.link .MT $(condition) : $(setup)mt -nologo ;
- toolset.flags intel-win.compile .MC $(condition) : $(setup)mc ;
- toolset.flags intel-win.compile .RC $(condition) : $(setup)rc ;
-
- local m = [ MATCH (.).* : $(version) ] ;
- local major = $(m[1]) ;
-
- local C++FLAGS ;
-
- C++FLAGS += /nologo ;
-
- # Reduce the number of spurious error messages
- C++FLAGS += /Qwn5 /Qwd985 ;
-
- # Enable ADL
- C++FLAGS += -Qoption,c,--arg_dep_lookup ; #"c" works for C++, too
-
- # Disable Microsoft "secure" overloads in Dinkumware libraries since they
- # cause compile errors with Intel versions 9 and 10.
- C++FLAGS += -D_SECURE_SCL=0 ;
-
- if $(major) > 5
- {
- C++FLAGS += /Zc:forScope ; # Add support for correct for loop scoping.
- }
-
- # Add options recognized only by intel7 and above.
- if $(major) >= 7
- {
- C++FLAGS += /Qansi_alias ;
- }
-
- if $(compatibility) = vc6
- {
- C++FLAGS +=
- # Emulate VC6
- /Qvc6
-
- # No wchar_t support in vc6 dinkum library. Furthermore, in vc6
- # compatibility-mode, wchar_t is not a distinct type from unsigned
- # short.
- -DBOOST_NO_INTRINSIC_WCHAR_T
- ;
- }
- else
- {
- if $(major) > 5
- {
- # Add support for wchar_t
- C++FLAGS += /Zc:wchar_t
- # Tell the dinkumware library about it.
- -D_NATIVE_WCHAR_T_DEFINED
- ;
- }
- }
-
- if $(compatibility) && $(compatibility) != native
- {
- C++FLAGS += /Q$(base-vc) ;
- }
- else
- {
- C++FLAGS +=
- -Qoption,cpp,--arg_dep_lookup
- # The following options were intended to disable the Intel compiler's
- # 'bug-emulation' mode, but were later reported to be causing ICE with
- # Intel-Win 9.0. It is not yet clear which options can be safely used.
- # -Qoption,cpp,--const_string_literals
- # -Qoption,cpp,--new_for_init
- # -Qoption,cpp,--no_implicit_typename
- # -Qoption,cpp,--no_friend_injection
- # -Qoption,cpp,--no_microsoft_bugs
- ;
- }
-
- toolset.flags intel-win CFLAGS $(condition) : $(C++FLAGS) ;
- # By default, when creating PCH, intel adds 'i' to the explicitly
- # specified name of the PCH file. Of course, Boost.Build is not
- # happy when compiler produces not the file it was asked for.
- # The option below stops this behaviour.
- toolset.flags intel-win CFLAGS : -Qpchi- ;
-
- if ! $(compatibility)
- {
- # If there's no backend version, assume 10.
- compatibility = vc10 ;
- }
-
- local extract-version = [ MATCH ^vc(.*) : $(compatibility) ] ;
- if ! $(extract-version)
- {
- errors.user-error "Invalid value for compatibility option:"
- $(compatibility) ;
- }
-
- # Depending on the settings, running of tests require some runtime DLLs.
- toolset.flags intel-win RUN_PATH $(condition) : $(root) ;
-
- msvc.configure-version-specific intel-win : $(extract-version[1]) : $(condition) ;
-}
-
-toolset.flags intel-win.link LIBRARY_OPTION <toolset>intel : "" ;
-
-toolset.flags intel-win YLOPTION ;
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/intel.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/intel.jam b/ext/kenlm/jam-files/boost-build/tools/intel.jam
deleted file mode 100644
index 67038aa..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/intel.jam
+++ /dev/null
@@ -1,34 +0,0 @@
-# Copyright Vladimir Prus 2004.
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt
-# or copy at http://www.boost.org/LICENSE_1_0.txt)
-
-# This is a generic 'intel' toolset. Depending on the current
-# system, it forwards either to 'intel-linux' or 'intel-win'
-# modules.
-
-import feature ;
-import os ;
-import toolset ;
-
-feature.extend toolset : intel ;
-feature.subfeature toolset intel : platform : : propagated link-incompatible ;
-
-rule init ( * : * )
-{
- if [ os.name ] = LINUX
- {
- toolset.using intel-linux :
- $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ;
- }
- else if [ os.name ] = MACOSX
- {
- toolset.using intel-darwin :
- $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ;
- }
- else
- {
- toolset.using intel-win :
- $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/jpeg.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/jpeg.jam b/ext/kenlm/jam-files/boost-build/tools/jpeg.jam
deleted file mode 100644
index 128ab63..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/jpeg.jam
+++ /dev/null
@@ -1,233 +0,0 @@
-# Copyright (c) 2010 Vladimir Prus.
-# Copyright (c) 2013 Steven Watanabe
-#
-# Use, modification and distribution is subject to the Boost Software
-# License Version 1.0. (See accompanying file LICENSE_1_0.txt or
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# Supports the libjpeg library
-#
-# After 'using libjpeg', the following targets are available:
-#
-# /libjpeg//libjpeg -- The libjpeg library
-
-import project ;
-import ac ;
-import errors ;
-import "class" : new ;
-import targets ;
-import path ;
-import modules ;
-import errors ;
-import indirect ;
-import property ;
-import property-set ;
-
-header = jconfig.h jdct.h jerror.h jinclude.h jmemsys.h jmorecfg.h jpegint.h jpeglib.h
- jversion.h ;
-
-names = libjpeg ;
-
-sources = jaricom.c jcapimin.c jcapistd.c jcarith.c jccoefct.c jccolor.c
- jcdctmgr.c jchuff.c jcinit.c jcmainct.c jcmarker.c jcmaster.c
- jcomapi.c jcparam.c jcprepct.c jcsample.c jctrans.c jdapimin.c
- jdapistd.c jdarith.c jdatadst.c jdatasrc.c jdcoefct.c jdcolor.c
- jddctmgr.c jdhuff.c jdinput.c jdmainct.c jdmarker.c jdmaster.c
- jdmerge.c jdpostct.c jdsample.c jdtrans.c jerror.c jfdctflt.c
- jfdctfst.c jfdctint.c jidctflt.c jidctfst.c jidctint.c jquant1.c ;
-
-library-id = 0 ;
-
-if --debug-configuration in [ modules.peek : ARGV ]
-{
- .debug = true ;
-}
-
-# Initializes the libjpeg library.
-#
-# libjpeg can be configured either to use pre-existing binaries
-# or to build the library from source.
-#
-# Options for configuring a prebuilt libjpeg::
-#
-# <search>
-# The directory containing the libjpeg binaries.
-# <name>
-# Overrides the default library name.
-# <include>
-# The directory containing the libjpeg headers.
-#
-# If none of these options is specified, then the environmental
-# variables LIBJPEG_LIBRARY_PATH, LIBJPEG_NAME, and LIBJPEG_INCLUDE will
-# be used instead.
-#
-# Options for building libjpeg from source::
-#
-# <source>
-# The libjpeg source directory. Defaults to the environmental variable
-# LIBJPEG_SOURCE.
-# <tag>
-# A rule which computes the actual name of the compiled
-# libraries based on the build properties. Ignored
-# when using precompiled binaries.
-# <build-name>
-# The base name to use for the compiled library. Ignored
-# when using precompiled binaries.
-#
-# Examples::
-#
-# # Find libjpeg in the default system location
-# using libjpeg ;
-# # Build libjpeg from source
-# using libjpeg : 8c : <source>/home/steven/libjpeg-8c ;
-# # Find libjpeg in /usr/local
-# using libjpeg : 8c
-# : <include>/usr/local/include <search>/usr/local/lib ;
-# # Build libjpeg from source for msvc and find
-# # prebuilt binaries for gcc.
-# using libjpeg : 8c : <source>C:/Devel/src/libjpeg-8c : <toolset>msvc ;
-# using libjpeg : 8c : : <toolset>gcc ;
-#
-rule init (
- version ?
- # The libjpeg version (currently ignored)
-
- : options *
- # A list of the options to use
-
- : requirements *
- # The requirements for the libjpeg target
-
- : is-default ?
- # Default configurations are only used when libjpeg
- # has not yet been configured.
- )
-{
- local caller = [ project.current ] ;
-
- if ! $(.initialized)
- {
- .initialized = true ;
-
- project.initialize $(__name__) ;
- .project = [ project.current ] ;
- project libjpeg ;
- }
-
- local library-path = [ property.select <search> : $(options) ] ;
- library-path = $(library-path:G=) ;
- local include-path = [ property.select <include> : $(options) ] ;
- include-path = $(include-path:G=) ;
- local source-path = [ property.select <source> : $(options) ] ;
- source-path = $(source-path:G=) ;
- local library-name = [ property.select <name> : $(options) ] ;
- library-name = $(library-name:G=) ;
- local tag = [ property.select <tag> : $(options) ] ;
- tag = $(tag:G=) ;
- local build-name = [ property.select <build-name> : $(options) ] ;
- build-name = $(build-name:G=) ;
-
- condition = [ property-set.create $(requirements) ] ;
- condition = [ property-set.create [ $(condition).base ] ] ;
-
- local no-build-from-source ;
- # Ignore environmental ZLIB_SOURCE if this initialization
- # requested to search for a specific pre-built library.
- if $(library-path) || $(include-path) || $(library-name)
- {
- if $(source-path) || $(tag) || $(build-name)
- {
- errors.user-error "incompatible options for libjpeg:"
- [ property.select <search> <include> <name> : $(options) ] "and"
- [ property.select <source> <tag> <build-name> : $(options) ] ;
- }
- else
- {
- no-build-from-source = true ;
- }
- }
-
- source-path ?= [ modules.peek : ZLIB_SOURCE ] ;
-
- if $(.configured.$(condition))
- {
- if $(is-default)
- {
- if $(.debug)
- {
- ECHO "notice: [libjpeg] libjpeg is already configured" ;
- }
- }
- else
- {
- errors.user-error "libjpeg is already configured" ;
- }
- return ;
- }
- else if $(source-path) && ! $(no-build-from-source)
- {
- build-name ?= z ;
- library-id = [ CALC $(library-id) + 1 ] ;
- tag = [ MATCH ^@?(.*)$ : $(tag) ] ;
- if $(tag) && ! [ MATCH ^([^%]*)%([^%]+)$ : $(tag) ]
- {
- tag = [ indirect.make $(tag) : [ $(caller).project-module ] ] ;
- }
- sources = [ path.glob $(source-path) : $(sources) ] ;
- if $(.debug)
- {
- ECHO "notice: [libjpeg] Building libjpeg from source as $(build-name)" ;
- if $(condition)
- {
- ECHO "notice: [libjpeg] Condition" [ $(condition).raw ] ;
- }
- if $(sources)
- {
- ECHO "notice: [libjpeg] found libjpeg source in $(source-path)" ;
- }
- else
- {
- ECHO "warning: [libjpeg] could not find libjpeg source in $(source-path)" ;
- }
- }
- local target ;
- if $(sources) {
- target = [ targets.create-typed-target LIB : $(.project)
- : $(build-name).$(library-id)
- : $(sources)
- : $(requirements)
- <tag>@$(tag)
- <include>$(source-path)
- <toolset>msvc:<define>_CRT_SECURE_NO_DEPRECATE
- <toolset>msvc:<define>_SCL_SECURE_NO_DEPRECATE
- <link>shared:<define>ZLIB_DLL
- :
- : <include>$(source-path) ] ;
- }
-
- local mt = [ new ac-library libjpeg : $(.project) : $(condition) ] ;
- $(mt).set-header $(header) ;
- $(mt).set-default-names $(names) ;
- if $(target)
- {
- $(mt).set-target $(target) ;
- }
- targets.main-target-alternative $(mt) ;
- } else {
- if $(.debug)
- {
- ECHO "notice: [libjpeg] Using pre-installed library" ;
- if $(condition)
- {
- ECHO "notice: [libjpeg] Condition" [ $(condition).raw ] ;
- }
- }
-
- local mt = [ new ac-library libjpeg : $(.project) : $(condition) :
- $(include-path) : $(library-path) : $(library-name) : $(root) ] ;
- $(mt).set-header $(header) ;
- $(mt).set-default-names $(names) ;
- targets.main-target-alternative $(mt) ;
- }
- .configured.$(condition) = true ;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/lex.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/lex.jam b/ext/kenlm/jam-files/boost-build/tools/lex.jam
deleted file mode 100644
index 75d6413..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/lex.jam
+++ /dev/null
@@ -1,33 +0,0 @@
-# Copyright 2003 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-import type ;
-import generators ;
-import feature ;
-import property ;
-
-
-feature.feature flex.prefix : : free ;
-type.register LEX : l ;
-type.register LEX++ : ll ;
-generators.register-standard lex.lex : LEX : C ;
-generators.register-standard lex.lex : LEX++ : CPP ;
-
-rule init ( )
-{
-}
-
-rule lex ( target : source : properties * )
-{
- local r = [ property.select flex.prefix : $(properties) ] ;
- if $(r)
- {
- PREFIX on $(<) = $(r:G=) ;
- }
-}
-
-actions lex
-{
- flex -P$(PREFIX) -o$(<) $(>)
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/make.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/make.jam b/ext/kenlm/jam-files/boost-build/tools/make.jam
deleted file mode 100644
index 40b59fa..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/make.jam
+++ /dev/null
@@ -1,63 +0,0 @@
-# Copyright 2003 Dave Abrahams
-# Copyright 2003 Douglas Gregor
-# Copyright 2006 Rene Rivera
-# Copyright 2002, 2003, 2004, 2005, 2006 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# This module defines the 'make' main target rule.
-
-import "class" : new ;
-import project ;
-import property-set ;
-import targets ;
-
-
-class make-target-class : basic-target
-{
- import "class" : new ;
- import type ;
- import virtual-target ;
-
- rule __init__ ( name : project : sources * : requirements *
- : default-build * : usage-requirements * )
- {
- basic-target.__init__ $(name) : $(project) : $(sources) :
- $(requirements) : $(default-build) : $(usage-requirements) ;
- }
-
- rule construct ( name : source-targets * : property-set )
- {
- local action-name = [ $(property-set).get <action> ] ;
- # 'm' will always be set -- we add '@' ourselves in the 'make' rule
- # below.
- local m = [ MATCH ^@(.*) : $(action-name) ] ;
-
- local a = [ new action $(source-targets) : $(m[1]) : $(property-set) ] ;
- local t = [ new file-target $(self.name) exact : [ type.type
- $(self.name) ] : $(self.project) : $(a) ] ;
- return [ property-set.empty ] [ virtual-target.register $(t) ] ;
- }
-}
-
-
-# Declares the 'make' main target.
-#
-rule make ( target-name : sources * : generating-rule + : requirements * :
- usage-requirements * )
-{
- # The '@' sign causes the feature.jam module to qualify rule name with the
- # module name of current project, if needed.
- local m = [ MATCH ^(@).* : $(generating-rule) ] ;
- if ! $(m)
- {
- generating-rule = @$(generating-rule) ;
- }
- targets.create-metatarget make-target-class : [ project.current ] :
- $(target-name) : $(sources) : $(requirements) <action>$(generating-rule)
- : : $(usage-requirements) ;
-}
-
-
-IMPORT $(__name__) : make : : make ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/mc.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/mc.jam b/ext/kenlm/jam-files/boost-build/tools/mc.jam
deleted file mode 100644
index 5783777..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/mc.jam
+++ /dev/null
@@ -1,44 +0,0 @@
-#~ Copyright 2005 Alexey Pakhunov.
-#~ Distributed under the Boost Software License, Version 1.0.
-#~ (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-# Support for Microsoft message compiler tool.
-# Notes:
-# - there's just message compiler tool, there's no tool for
-# extracting message strings from sources
-# - This file allows to use Microsoft message compiler
-# with any toolset. In msvc.jam, there's more specific
-# message compiling action.
-
-import common ;
-import generators ;
-import feature : feature get-values ;
-import toolset : flags ;
-import type ;
-import rc ;
-
-rule init ( )
-{
-}
-
-type.register MC : mc ;
-
-
-# Command line options
-feature mc-input-encoding : ansi unicode : free ;
-feature mc-output-encoding : unicode ansi : free ;
-feature mc-set-customer-bit : no yes : free ;
-
-flags mc.compile MCFLAGS <mc-input-encoding>ansi : -a ;
-flags mc.compile MCFLAGS <mc-input-encoding>unicode : -u ;
-flags mc.compile MCFLAGS <mc-output-encoding>ansi : -A ;
-flags mc.compile MCFLAGS <mc-output-encoding>unicode : -U ;
-flags mc.compile MCFLAGS <mc-set-customer-bit>no : ;
-flags mc.compile MCFLAGS <mc-set-customer-bit>yes : -c ;
-
-generators.register-standard mc.compile : MC : H RC ;
-
-actions compile
-{
- mc $(MCFLAGS) -h "$(<[1]:DW)" -r "$(<[2]:DW)" "$(>:W)"
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/message.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/message.jam b/ext/kenlm/jam-files/boost-build/tools/message.jam
deleted file mode 100644
index 672b6e0..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/message.jam
+++ /dev/null
@@ -1,62 +0,0 @@
-# Copyright 2008 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-# Defines main target type 'message', that prints a message when built for the
-# first time.
-
-import project ;
-import "class" : new ;
-import targets ;
-import property-set ;
-
-class message-target-class : basic-target
-{
- rule set-message ( * )
- {
- self.1 = $(1) ;
- self.2 = $(2) ;
- self.3 = $(3) ;
- self.4 = $(4) ;
- self.5 = $(5) ;
- self.6 = $(6) ;
- self.7 = $(7) ;
- self.8 = $(8) ;
- self.9 = $(9) ;
- self.built = ;
- }
-
- rule construct ( name : source-targets * : property-set )
- {
- if ! $(self.built)
- {
- for i in 1 2 3 4 5 6 7 8 9
- {
- if $(self.$(i))
- {
- ECHO $(self.$(i)) ;
- }
- }
- self.built = 1 ;
- }
-
- return [ property-set.empty ] ;
- }
-}
-
-
-rule message ( name : * )
-{
- local project = [ project.current ] ;
-
- local result = [ targets.main-target-alternative
- [ new message-target-class $(name) : $(project)
- : [ targets.main-target-sources : $(name) ]
- : [ targets.main-target-requirements : $(project) ]
- : [ targets.main-target-default-build : $(project) ]
- : [ targets.main-target-usage-requirements : $(project) ]
- ] ] ;
- $(result).set-message $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ;
- return $(result) ;
-}
-IMPORT $(__name__) : message : : message ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/midl.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/midl.jam b/ext/kenlm/jam-files/boost-build/tools/midl.jam
deleted file mode 100644
index 0aa5dda..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/midl.jam
+++ /dev/null
@@ -1,142 +0,0 @@
-# Copyright (c) 2005 Alexey Pakhunov.
-#
-# Use, modification and distribution is subject to the Boost Software
-# License Version 1.0. (See accompanying file LICENSE_1_0.txt or
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# Microsoft Interface Definition Language (MIDL) related routines
-
-import common ;
-import generators ;
-import feature : feature get-values ;
-import os ;
-import scanner ;
-import toolset : flags ;
-import type ;
-
-rule init ( )
-{
-}
-
-type.register IDL : idl ;
-
-# A type library (.tlb) is generated by MIDL compiler and can be included
-# to resources of an application (.rc). In order to be found by a resource
-# compiler its target type should be derived from 'H' - otherwise
-# the property '<implicit-dependency>' will be ignored.
-type.register MSTYPELIB : tlb : H ;
-
-
-# Register scanner for MIDL files
-class midl-scanner : scanner
-{
- import path property-set regex scanner type virtual-target ;
-
- rule __init__ ( includes * )
- {
- scanner.__init__ ;
-
- self.includes = $(includes) ;
-
- # List of quoted strings
- self.re-strings = "[ \t]*\"([^\"]*)\"([ \t]*,[ \t]*\"([^\"]*)\")*[ \t]*" ;
-
- # 'import' and 'importlib' directives
- self.re-import = "import"$(self.re-strings)"[ \t]*;" ;
- self.re-importlib = "importlib[ \t]*[(]"$(self.re-strings)"[)][ \t]*;" ;
-
- # C preprocessor 'include' directive
- self.re-include-angle = "#[ \t]*include[ \t]*<(.*)>" ;
- self.re-include-quoted = "#[ \t]*include[ \t]*\"(.*)\"" ;
- }
-
- rule pattern ( )
- {
- # Match '#include', 'import' and 'importlib' directives
- return "((#[ \t]*include|import(lib)?).+(<(.*)>|\"(.*)\").+)" ;
- }
-
- rule process ( target : matches * : binding )
- {
- local included-angle = [ regex.transform $(matches) : $(self.re-include-angle) : 1 ] ;
- local included-quoted = [ regex.transform $(matches) : $(self.re-include-quoted) : 1 ] ;
- local imported = [ regex.transform $(matches) : $(self.re-import) : 1 3 ] ;
- local imported_tlbs = [ regex.transform $(matches) : $(self.re-importlib) : 1 3 ] ;
-
- # CONSIDER: the new scoping rule seem to defeat "on target" variables.
- local g = [ on $(target) return $(HDRGRIST) ] ;
- local b = [ NORMALIZE_PATH $(binding:D) ] ;
-
- # Attach binding of including file to included targets.
- # When target is directly created from virtual target
- # this extra information is unnecessary. But in other
- # cases, it allows to distinguish between two headers of the
- # same name included from different places.
- local g2 = $(g)"#"$(b) ;
-
- included-angle = $(included-angle:G=$(g)) ;
- included-quoted = $(included-quoted:G=$(g2)) ;
- imported = $(imported:G=$(g2)) ;
- imported_tlbs = $(imported_tlbs:G=$(g2)) ;
-
- local all = $(included-angle) $(included-quoted) $(imported) ;
-
- INCLUDES $(target) : $(all) ;
- DEPENDS $(target) : $(imported_tlbs) ;
- NOCARE $(all) $(imported_tlbs) ;
- SEARCH on $(included-angle) = $(self.includes:G=) ;
- SEARCH on $(included-quoted) = $(b) $(self.includes:G=) ;
- SEARCH on $(imported) = $(b) $(self.includes:G=) ;
- SEARCH on $(imported_tlbs) = $(b) $(self.includes:G=) ;
-
- scanner.propagate
- [ type.get-scanner CPP : [ property-set.create $(self.includes) ] ] :
- $(included-angle) $(included-quoted) : $(target) ;
-
- scanner.propagate $(__name__) : $(imported) : $(target) ;
- }
-}
-
-scanner.register midl-scanner : include ;
-type.set-scanner IDL : midl-scanner ;
-
-
-# Command line options
-feature midl-stubless-proxy : yes no : propagated ;
-feature midl-robust : yes no : propagated ;
-
-flags midl.compile.idl MIDLFLAGS <midl-stubless-proxy>yes : /Oicf ;
-flags midl.compile.idl MIDLFLAGS <midl-stubless-proxy>no : /Oic ;
-flags midl.compile.idl MIDLFLAGS <midl-robust>yes : /robust ;
-flags midl.compile.idl MIDLFLAGS <midl-robust>no : /no_robust ;
-
-# Architecture-specific options
-architecture-x86 = <architecture> <architecture>x86 ;
-address-model-32 = <address-model> <address-model>32 ;
-address-model-64 = <address-model> <address-model>64 ;
-
-flags midl.compile.idl MIDLFLAGS $(architecture-x86)/$(address-model-32) : /win32 ;
-flags midl.compile.idl MIDLFLAGS $(architecture-x86)/<address-model>64 : /x64 ;
-flags midl.compile.idl MIDLFLAGS <architecture>ia64/$(address-model-64) : /ia64 ;
-
-
-flags midl.compile.idl DEFINES <define> ;
-flags midl.compile.idl UNDEFS <undef> ;
-flags midl.compile.idl INCLUDES <include> ;
-
-
-generators.register-c-compiler midl.compile.idl : IDL : MSTYPELIB H C(%_i) C(%_proxy) C(%_dlldata) ;
-
-
-# MIDL does not always generate '%_proxy.c' and '%_dlldata.c'. This behavior
-# depends on contents of the source IDL file. Calling TOUCH_FILE below ensures
-# that both files will be created so bjam will not try to recreate them
-# constantly.
-TOUCH_FILE = [ common.file-touch-command ] ;
-
-actions compile.idl
-{
- midl /nologo @"@($(<[1]:W).rsp:E=$(nl)"$(>:W)" $(nl)-D$(DEFINES) $(nl)"-I$(INCLUDES)" $(nl)-U$(UNDEFS) $(nl)$(MIDLFLAGS) $(nl)/tlb "$(<[1]:W)" $(nl)/h "$(<[2]:W)" $(nl)/iid "$(<[3]:W)" $(nl)/proxy "$(<[4]:W)" $(nl)/dlldata "$(<[5]:W)")"
- $(TOUCH_FILE) "$(<[4]:W)"
- $(TOUCH_FILE) "$(<[5]:W)"
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/mipspro.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/mipspro.jam b/ext/kenlm/jam-files/boost-build/tools/mipspro.jam
deleted file mode 100644
index 417eaef..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/mipspro.jam
+++ /dev/null
@@ -1,145 +0,0 @@
-# Copyright Noel Belcourt 2007.
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-import property ;
-import generators ;
-import os ;
-import toolset : flags ;
-import feature ;
-import fortran ;
-import type ;
-import common ;
-
-feature.extend toolset : mipspro ;
-toolset.inherit mipspro : unix ;
-generators.override mipspro.prebuilt : builtin.lib-generator ;
-generators.override mipspro.searched-lib-generator : searched-lib-generator ;
-
-# Documentation and toolchain description located
-# http://www.sgi.com/products/software/irix/tools/
-
-rule init ( version ? : command * : options * )
-{
- local condition = [
- common.check-init-parameters mipspro : version $(version) ] ;
-
- command = [ common.get-invocation-command mipspro : CC : $(command) ] ;
-
- common.handle-options mipspro : $(condition) : $(command) : $(options) ;
-
- command_c = $(command_c[1--2]) $(command[-1]:B=cc) ;
-
- toolset.flags mipspro CONFIG_C_COMMAND $(condition) : $(command_c) ;
-
- # fortran support
- local command = [
- common.get-invocation-command mipspro : f77 : $(command) : $(install_dir) ] ;
-
- command_f = $(command_f[1--2]) $(command[-1]:B=f77) ;
- toolset.flags mipspro CONFIG_F_COMMAND $(condition) : $(command_f) ;
-
- # set link flags
- flags mipspro.link FINDLIBS-ST : [
- feature.get-values <find-static-library> : $(options) ] : unchecked ;
-
- flags mipspro.link FINDLIBS-SA : [
- feature.get-values <find-shared-library> : $(options) ] : unchecked ;
-}
-
-# Declare generators
-generators.register-c-compiler mipspro.compile.c : C : OBJ : <toolset>mipspro ;
-generators.register-c-compiler mipspro.compile.c++ : CPP : OBJ : <toolset>mipspro ;
-generators.register-fortran-compiler mipspro.compile.fortran : FORTRAN : OBJ : <toolset>mipspro ;
-
-cpu-arch-32 =
- <architecture>/<address-model>
- <architecture>/<address-model>32 ;
-
-cpu-arch-64 =
- <architecture>/<address-model>64 ;
-
-flags mipspro.compile OPTIONS $(cpu-arch-32) : -n32 ;
-flags mipspro.compile OPTIONS $(cpu-arch-64) : -64 ;
-
-# Declare flags and actions for compilation
-flags mipspro.compile OPTIONS <debug-symbols>on : -g ;
-# flags mipspro.compile OPTIONS <profiling>on : -xprofile=tcov ;
-flags mipspro.compile OPTIONS <warnings>off : -w ;
-flags mipspro.compile OPTIONS <warnings>on : -ansiW -diag_suppress 1429 ; # suppress long long is nonstandard warning
-flags mipspro.compile OPTIONS <warnings>all : -fullwarn ;
-flags mipspro.compile OPTIONS <optimization>speed : -Ofast ;
-flags mipspro.compile OPTIONS <optimization>space : -O2 ;
-flags mipspro.compile OPTIONS <cflags> : -LANG:std ;
-flags mipspro.compile.c++ OPTIONS <inlining>off : -INLINE:none ;
-flags mipspro.compile.c++ OPTIONS <cxxflags> ;
-flags mipspro.compile DEFINES <define> ;
-flags mipspro.compile INCLUDES <include> ;
-
-
-flags mipspro.compile.fortran OPTIONS <fflags> ;
-
-actions compile.c
-{
- "$(CONFIG_C_COMMAND)" $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
-}
-
-actions compile.c++
-{
- "$(CONFIG_COMMAND)" -FE:template_in_elf_section -ptused $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
-}
-
-actions compile.fortran
-{
- "$(CONFIG_F_COMMAND)" $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
-}
-
-# Declare flags and actions for linking
-flags mipspro.link OPTIONS <debug-symbols>on : -g ;
-# Strip the binary when no debugging is needed
-# flags mipspro.link OPTIONS <debug-symbols>off : -s ;
-# flags mipspro.link OPTIONS <profiling>on : -xprofile=tcov ;
-# flags mipspro.link OPTIONS <threading>multi : -mt ;
-
-flags mipspro.link OPTIONS $(cpu-arch-32) : -n32 ;
-flags mipspro.link OPTIONS $(cpu-arch-64) : -64 ;
-
-flags mipspro.link OPTIONS <optimization>speed : -Ofast ;
-flags mipspro.link OPTIONS <optimization>space : -O2 ;
-flags mipspro.link OPTIONS <linkflags> ;
-flags mipspro.link LINKPATH <library-path> ;
-flags mipspro.link FINDLIBS-ST <find-static-library> ;
-flags mipspro.link FINDLIBS-SA <find-shared-library> ;
-flags mipspro.link FINDLIBS-SA <threading>multi : pthread ;
-flags mipspro.link LIBRARIES <library-file> ;
-flags mipspro.link LINK-RUNTIME <runtime-link>static : static ;
-flags mipspro.link LINK-RUNTIME <runtime-link>shared : dynamic ;
-flags mipspro.link RPATH <dll-path> ;
-
-rule link ( targets * : sources * : properties * )
-{
- SPACE on $(targets) = " " ;
-}
-
-actions link bind LIBRARIES
-{
- "$(CONFIG_COMMAND)" -FE:template_in_elf_section -ptused $(OPTIONS) -L"$(LINKPATH)" -R"$(RPATH)" -o "$(<)" "$(>)" "$(LIBRARIES)" -Bdynamic -l$(FINDLIBS-SA) -Bstatic -l$(FINDLIBS-ST) -B$(LINK-RUNTIME) -lm
-}
-
-# Slight mods for dlls
-rule link.dll ( targets * : sources * : properties * )
-{
- SPACE on $(targets) = " " ;
-}
-
-actions link.dll bind LIBRARIES
-{
- "$(CONFIG_COMMAND)" $(OPTIONS) -L"$(LINKPATH)" -R"$(RPATH)" -o "$(<)" "$(>)" "$(LIBRARIES)" -Bdynamic -l$(FINDLIBS-SA) -Bstatic -l$(FINDLIBS-ST) -B$(LINK-RUNTIME)
-}
-
-# Declare action for creating static libraries
-actions piecemeal archive
-{
- ar -cr "$(<)" "$(>)"
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/mpi.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/mpi.jam b/ext/kenlm/jam-files/boost-build/tools/mpi.jam
deleted file mode 100644
index 0fe490b..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/mpi.jam
+++ /dev/null
@@ -1,583 +0,0 @@
-# Support for the Message Passing Interface (MPI)
-#
-# (C) Copyright 2005, 2006 Trustees of Indiana University
-# (C) Copyright 2005 Douglas Gregor
-#
-# Distributed under the Boost Software License, Version 1.0. (See accompanying
-# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt.)
-#
-# Authors: Douglas Gregor
-# Andrew Lumsdaine
-#
-# ==== MPI Configuration ====
-#
-# For many users, MPI support can be enabled simply by adding the following
-# line to your user-config.jam file:
-#
-# using mpi ;
-#
-# This should auto-detect MPI settings based on the MPI wrapper compiler in
-# your path, e.g., "mpic++". If the wrapper compiler is not in your path, or
-# has a different name, you can pass the name of the wrapper compiler as the
-# first argument to the mpi module:
-#
-# using mpi : /opt/mpich2-1.0.4/bin/mpiCC ;
-#
-# If your MPI implementation does not have a wrapper compiler, or the MPI
-# auto-detection code does not work with your MPI's wrapper compiler,
-# you can pass MPI-related options explicitly via the second parameter to the
-# mpi module:
-#
-# using mpi : : <find-shared-library>lammpio <find-shared-library>lammpi++
-# <find-shared-library>mpi <find-shared-library>lam
-# <find-shared-library>dl ;
-#
-# To see the results of MPI auto-detection, pass "--debug-configuration" on
-# the bjam command line.
-#
-# The (optional) fourth argument configures Boost.MPI for running
-# regression tests. These parameters specify the executable used to
-# launch jobs (default: "mpirun") followed by any necessary arguments
-# to this to run tests and tell the program to expect the number of
-# processors to follow (default: "-np"). With the default parameters,
-# for instance, the test harness will execute, e.g.,
-#
-# mpirun -np 4 all_gather_test
-#
-# ==== Linking Against the MPI Libraries ===
-#
-# To link against the MPI libraries, import the "mpi" module and add the
-# following requirement to your target:
-#
-# <library>/mpi//mpi
-#
-# Since MPI support is not always available, you should check
-# "mpi.configured" before trying to link against the MPI libraries.
-
-import "class" : new ;
-import common ;
-import feature : feature ;
-import generators ;
-import os ;
-import project ;
-import property ;
-import testing ;
-import toolset ;
-import type ;
-import path ;
-
-# Make this module a project
-project.initialize $(__name__) ;
-project mpi ;
-
-if [ MATCH (--debug-configuration) : [ modules.peek : ARGV ] ]
-{
- .debug-configuration = true ;
-}
-
-# Assuming the first part of the command line is the given prefix
-# followed by some non-empty value, remove the first argument. Returns
-# either nothing (if there was no prefix or no value) or a pair
-#
-# <name>value rest-of-cmdline
-#
-# This is a subroutine of cmdline_to_features
-rule add_feature ( prefix name cmdline )
-{
- local match = [ MATCH "^$(prefix)([^\" ]+|\"[^\"]+\") *(.*)$" : $(cmdline) ] ;
-
- # If there was no value associated with the prefix, abort
- if ! $(match) {
- return ;
- }
-
- local value = $(match[1]) ;
-
- if [ MATCH " +" : $(value) ] {
- value = "\"$(value)\"" ;
- }
-
- return "<$(name)>$(value)" $(match[2]) ;
-}
-
-# Strip any end-of-line characters off the given string and return the
-# result.
-rule strip-eol ( string )
-{
- local match = [ MATCH "^(([A-Za-z0-9~`\.!@#$%^&*()_+={};:'\",.<>/?\\| -]|[|])*).*$" : $(string) ] ;
-
- if $(match)
- {
- return $(match[1]) ;
- }
- else
- {
- return $(string) ;
- }
-}
-
-# Split a command-line into a set of features. Certain kinds of
-# compiler flags are recognized (e.g., -I, -D, -L, -l) and replaced
-# with their Boost.Build equivalents (e.g., <include>, <define>,
-# <library-path>, <find-library>). All other arguments are introduced
-# using the features in the unknown-features parameter, because we
-# don't know how to deal with them. For instance, if your compile and
-# correct. The incoming command line should be a string starting with
-# an executable (e.g., g++ -I/include/path") and may contain any
-# number of command-line arguments thereafter. The result is a list of
-# features corresponding to the given command line, ignoring the
-# executable.
-rule cmdline_to_features ( cmdline : unknown-features ? )
-{
- local executable ;
- local features ;
- local otherflags ;
- local result ;
-
- unknown-features ?= <cxxflags> <linkflags> ;
-
- # Pull the executable out of the command line. At this point, the
- # executable is just thrown away.
- local match = [ MATCH "^([^\" ]+|\"[^\"]+\") *(.*)$" : $(cmdline) ] ;
- executable = $(match[1]) ;
- cmdline = $(match[2]) ;
-
- # List the prefix/feature pairs that we will be able to transform.
- # Every kind of parameter not mentioned here will be placed in both
- # cxxflags and linkflags, because we don't know where they should go.
- local feature_kinds-D = "define" ;
- local feature_kinds-I = "include" ;
- local feature_kinds-L = "library-path" ;
- local feature_kinds-l = "find-shared-library" ;
-
- while $(cmdline) {
-
- # Check for one of the feature prefixes we know about. If we
- # find one (and the associated value is nonempty), convert it
- # into a feature.
- local match = [ MATCH "^(-.)(.*)" : $(cmdline) ] ;
- local matched ;
- if $(match) && $(match[2]) {
- local prefix = $(match[1]) ;
- if $(feature_kinds$(prefix)) {
- local name = $(feature_kinds$(prefix)) ;
- local add = [ add_feature $(prefix) $(name) $(cmdline) ] ;
-
- if $(add) {
-
- if $(add[1]) = <find-shared-library>pthread
- {
- # Uhm. It's not really nice that this MPI implementation
- # uses -lpthread as opposed to -pthread. We do want to
- # set <threading>multi, instead of -lpthread.
- result += "<threading>multi" ;
- MPI_EXTRA_REQUIREMENTS += "<threading>multi" ;
- }
- else
- {
- result += $(add[1]) ;
- }
-
- cmdline = $(add[2]) ;
- matched = yes ;
- }
- }
- }
-
- # If we haven't matched a feature prefix, just grab the command-line
- # argument itself. If we can map this argument to a feature
- # (e.g., -pthread -> <threading>multi), then do so; otherwise,
- # and add it to the list of "other" flags that we don't
- # understand.
- if ! $(matched) {
- match = [ MATCH "^([^\" ]+|\"[^\"]+\") *(.*)$" : $(cmdline) ] ;
- local value = $(match[1]) ;
- cmdline = $(match[2]) ;
-
- # Check for multithreading support
- if $(value) = "-pthread" || $(value) = "-pthreads"
- {
- result += "<threading>multi" ;
-
- # DPG: This is a hack intended to work around a BBv2 bug where
- # requirements propagated from libraries are not checked for
- # conflicts when BBv2 determines which "common" properties to
- # apply to a target. In our case, the <threading>single property
- # gets propagated from the common properties to Boost.MPI
- # targets, even though <threading>multi is in the usage
- # requirements of <library>/mpi//mpi.
- MPI_EXTRA_REQUIREMENTS += "<threading>multi" ;
- }
- else if [ MATCH "(.*[a-zA-Z0-9<>?-].*)" : $(value) ] {
- otherflags += $(value) ;
- }
- }
- }
-
- # If there are other flags that we don't understand, add them to the
- # result as both <cxxflags> and <linkflags>
- if $(otherflags) {
- for unknown in $(unknown-features)
- {
- result += "$(unknown)$(otherflags:J= )" ;
- }
- }
-
- return $(result) ;
-}
-
-# Determine if it is safe to execute the given shell command by trying
-# to execute it and determining whether the exit code is zero or
-# not. Returns true for an exit code of zero, false otherwise.
-local rule safe-shell-command ( cmdline )
-{
- local result = [ SHELL "$(cmdline) > /dev/null 2>/dev/null; if [ "$?" -eq "0" ]; then echo SSCOK; fi" ] ;
- return [ MATCH ".*(SSCOK).*" : $(result) ] ;
-}
-
-# Initialize the MPI module.
-rule init ( mpicxx ? : options * : mpirun-with-options * )
-{
- if ! $(options) && $(.debug-configuration)
- {
- ECHO "===============MPI Auto-configuration===============" ;
- }
-
- if ! $(mpicxx) && [ os.on-windows ]
- {
- # Try to auto-configure to the Microsoft Compute Cluster Pack
- local cluster_pack_path_native = "C:\\Program Files\\Microsoft Compute Cluster Pack" ;
- local cluster_pack_path = [ path.make $(cluster_pack_path_native) ] ;
- if [ GLOB $(cluster_pack_path_native)\\Include : mpi.h ]
- {
- if $(.debug-configuration)
- {
- ECHO "Found Microsoft Compute Cluster Pack: $(cluster_pack_path_native)" ;
- }
-
- # Pick up either the 32-bit or 64-bit library, depending on which address
- # model the user has selected. Default to 32-bit.
- options = <include>$(cluster_pack_path)/Include
- <address-model>64:<library-path>$(cluster_pack_path)/Lib/amd64
- <library-path>$(cluster_pack_path)/Lib/i386
- <find-static-library>msmpi
- <toolset>msvc:<define>_SECURE_SCL=0
- ;
-
- # Setup the "mpirun" equivalent (mpiexec)
- .mpirun = "\"$(cluster_pack_path_native)\\Bin\\mpiexec.exe"\" ;
- .mpirun_flags = -n ;
- }
- else if $(.debug-configuration)
- {
- ECHO "Did not find Microsoft Compute Cluster Pack in $(cluster_pack_path_native)." ;
- }
- }
-
- if ! $(options)
- {
- # Try to auto-detect options based on the wrapper compiler
- local command = [ common.get-invocation-command mpi : mpic++ : $(mpicxx) ] ;
-
- if ! $(mpicxx) && ! $(command)
- {
- # Try "mpiCC", which is used by MPICH
- command = [ common.get-invocation-command mpi : mpiCC ] ;
- }
-
- if ! $(mpicxx) && ! $(command)
- {
- # Try "mpicxx", which is used by OpenMPI and MPICH2
- command = [ common.get-invocation-command mpi : mpicxx ] ;
- }
-
- local result ;
- local compile_flags ;
- local link_flags ;
-
- if ! $(command)
- {
- # Do nothing: we'll complain later
- }
- # OpenMPI and newer versions of LAM-MPI have -showme:compile and
- # -showme:link.
- else if [ safe-shell-command "$(command) -showme:compile" ] &&
- [ safe-shell-command "$(command) -showme:link" ]
- {
- if $(.debug-configuration)
- {
- ECHO "Found recent LAM-MPI or Open MPI wrapper compiler: $(command)" ;
- }
-
- compile_flags = [ SHELL "$(command) -showme:compile" ] ;
- link_flags = [ SHELL "$(command) -showme:link" ] ;
-
- # Prepend COMPILER as the executable name, to match the format of
- # other compilation commands.
- compile_flags = "COMPILER $(compile_flags)" ;
- link_flags = "COMPILER $(link_flags)" ;
- }
- # Look for LAM-MPI's -showme
- else if [ safe-shell-command "$(command) -showme" ]
- {
- if $(.debug-configuration)
- {
- ECHO "Found older LAM-MPI wrapper compiler: $(command)" ;
- }
-
- result = [ SHELL "$(command) -showme" ] ;
- }
- # Look for MPICH
- else if [ safe-shell-command "$(command) -show" ]
- {
- if $(.debug-configuration)
- {
- ECHO "Found MPICH wrapper compiler: $(command)" ;
- }
- compile_flags = [ SHELL "$(command) -compile_info" ] ;
- link_flags = [ SHELL "$(command) -link_info" ] ;
- }
- # Sun HPC and Ibm POE
- else if [ SHELL "$(command) -v 2>/dev/null" ]
- {
- compile_flags = [ SHELL "$(command) -c -v -xtarget=native64 2>/dev/null" ] ;
-
- local back = [ MATCH "--------------------(.*)" : $(compile_flags) ] ;
- if $(back)
- {
- # Sun HPC
- if $(.debug-configuration)
- {
- ECHO "Found Sun MPI wrapper compiler: $(command)" ;
- }
-
- compile_flags = [ MATCH "(.*)--------------------" : $(back) ] ;
- compile_flags = [ MATCH "(.*)-v" : $(compile_flags) ] ;
- link_flags = [ SHELL "$(command) -v -xtarget=native64 2>/dev/null" ] ;
- link_flags = [ MATCH "--------------------(.*)" : $(link_flags) ] ;
- link_flags = [ MATCH "(.*)--------------------" : $(link_flags) ] ;
-
- # strip out -v from compile options
- local front = [ MATCH "(.*)-v" : $(link_flags) ] ;
- local back = [ MATCH "-v(.*)" : $(link_flags) ] ;
- link_flags = "$(front) $(back)" ;
- front = [ MATCH "(.*)-xtarget=native64" : $(link_flags) ] ;
- back = [ MATCH "-xtarget=native64(.*)" : $(link_flags) ] ;
- link_flags = "$(front) $(back)" ;
- }
- else
- {
- # Ibm POE
- if $(.debug-configuration)
- {
- ECHO "Found IBM MPI wrapper compiler: $(command)" ;
- }
-
- #
- compile_flags = [ SHELL "$(command) -c -v 2>/dev/null" ] ;
- compile_flags = [ MATCH "(.*)exec: export.*" : $(compile_flags) ] ;
- local front = [ MATCH "(.*)-v" : $(compile_flags) ] ;
- local back = [ MATCH "-v(.*)" : $(compile_flags) ] ;
- compile_flags = "$(front) $(back)" ;
- front = [ MATCH "(.*)-c" : $(compile_flags) ] ;
- back = [ MATCH "-c(.*)" : $(compile_flags) ] ;
- compile_flags = "$(front) $(back)" ;
- link_flags = $(compile_flags) ;
-
- # get location of mpif.h from mpxlf
- local f_flags = [ SHELL "mpxlf -v 2>/dev/null" ] ;
- f_flags = [ MATCH "(.*)exec: export.*" : $(f_flags) ] ;
- front = [ MATCH "(.*)-v" : $(f_flags) ] ;
- back = [ MATCH "-v(.*)" : $(f_flags) ] ;
- f_flags = "$(front) $(back)" ;
- f_flags = [ MATCH "xlf_r(.*)" : $(f_flags) ] ;
- f_flags = [ MATCH "-F:mpxlf_r(.*)" : $(f_flags) ] ;
- compile_flags = [ strip-eol $(compile_flags) ] ;
- compile_flags = "$(compile_flags) $(f_flags)" ;
- }
- }
-
- if $(result) || $(compile_flags) && $(link_flags)
- {
- if $(result)
- {
- result = [ strip-eol $(result) ] ;
- options = [ cmdline_to_features $(result) ] ;
- }
- else
- {
- compile_flags = [ strip-eol $(compile_flags) ] ;
- link_flags = [ strip-eol $(link_flags) ] ;
-
- # Separately process compilation and link features, then combine
- # them at the end.
- local compile_features = [ cmdline_to_features $(compile_flags)
- : "<cxxflags>" ] ;
- local link_features = [ cmdline_to_features $(link_flags)
- : "<linkflags>" ] ;
- options = $(compile_features) $(link_features) ;
- }
-
- # If requested, display MPI configuration information.
- if $(.debug-configuration)
- {
- if $(result)
- {
- ECHO " Wrapper compiler command line: $(result)" ;
- }
- else
- {
- local match = [ MATCH "^([^\" ]+|\"[^\"]+\") *(.*)$"
- : $(compile_flags) ] ;
- ECHO "MPI compilation flags: $(match[2])" ;
- local match = [ MATCH "^([^\" ]+|\"[^\"]+\") *(.*)$"
- : $(link_flags) ] ;
- ECHO "MPI link flags: $(match[2])" ;
- }
- }
- }
- else
- {
- if $(command)
- {
- ECHO "MPI auto-detection failed: unknown wrapper compiler $(command)" ;
- ECHO "Please report this error to the Boost mailing list: http://www.boost.org" ;
- }
- else if $(mpicxx)
- {
- ECHO "MPI auto-detection failed: unable to find wrapper compiler $(mpicxx)" ;
- }
- else
- {
- ECHO "MPI auto-detection failed: unable to find wrapper compiler `mpic++' or `mpiCC'" ;
- }
- ECHO "You will need to manually configure MPI support." ;
- }
-
- }
-
- # Find mpirun (or its equivalent) and its flags
- if ! $(.mpirun)
- {
- .mpirun =
- [ common.get-invocation-command mpi : mpirun : $(mpirun-with-options[1]) ] ;
- .mpirun_flags = $(mpirun-with-options[2-]) ;
- .mpirun_flags ?= -np ;
- }
-
- if $(.debug-configuration)
- {
- if $(options)
- {
- echo "MPI build features: " ;
- ECHO $(options) ;
- }
-
- if $(.mpirun)
- {
- echo "MPI launcher: $(.mpirun) $(.mpirun_flags)" ;
- }
-
- ECHO "====================================================" ;
- }
-
- if $(options)
- {
- .configured = true ;
-
- # Set up the "mpi" alias
- alias mpi : : : : $(options) ;
- }
-}
-
-# States whether MPI has bee configured
-rule configured ( )
-{
- return $(.configured) ;
-}
-
-# Returs the "extra" requirements needed to build MPI. These requirements are
-# part of the /mpi//mpi library target, but they need to be added to anything
-# that uses MPI directly to work around bugs in BBv2's propagation of
-# requirements.
-rule extra-requirements ( )
-{
- return $(MPI_EXTRA_REQUIREMENTS) ;
-}
-
-# Support for testing; borrowed from Python
-type.register RUN_MPI_OUTPUT ;
-type.register RUN_MPI : : TEST ;
-
-class mpi-test-generator : generator
-{
- import property-set ;
-
- rule __init__ ( * : * )
- {
- generator.__init__ $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ;
- self.composing = true ;
- }
-
- rule run ( project name ? : property-set : sources * : multiple ? )
- {
- # Generate an executable from the sources. This is the executable we will run.
- local executable =
- [ generators.construct $(project) $(name) : EXE : $(property-set) : $(sources) ] ;
-
- result =
- [ construct-result $(executable[2-]) : $(project) $(name)-run : $(property-set) ] ;
- }
-}
-
-# Use mpi-test-generator to generate MPI tests from sources
-generators.register
- [ new mpi-test-generator mpi.capture-output : : RUN_MPI_OUTPUT ] ;
-
-generators.register-standard testing.expect-success
- : RUN_MPI_OUTPUT : RUN_MPI ;
-
-# The number of processes to spawn when executing an MPI test.
-feature mpi:processes : : free incidental ;
-
-# The flag settings on testing.capture-output do not
-# apply to mpi.capture output at the moment.
-# Redo this explicitly.
-toolset.flags mpi.capture-output ARGS <testing.arg> ;
-rule capture-output ( target : sources * : properties * )
-{
- # Use the standard capture-output rule to run the tests
- testing.capture-output $(target) : $(sources[1]) : $(properties) ;
-
- # Determine the number of processes we should run on.
- local num_processes = [ property.select <mpi:processes> : $(properties) ] ;
- num_processes = $(num_processes:G=) ;
-
- # serialize the MPI tests to avoid overloading systems
- JAM_SEMAPHORE on $(target) = <s>mpi-run-semaphore ;
-
- # We launch MPI processes using the "mpirun" equivalent specified by the user.
- LAUNCHER on $(target) =
- [ on $(target) return $(.mpirun) $(.mpirun_flags) $(num_processes) ] ;
-}
-
-# Creates a set of test cases to be run through the MPI launcher. The name, sources,
-# and requirements are the same as for any other test generator. However, schedule is
-# a list of numbers, which indicates how many processes each test run will use. For
-# example, passing 1 2 7 will run the test with 1 process, then 2 processes, then 7
-# 7 processes. The name provided is just the base name: the actual tests will be
-# the name followed by a hypen, then the number of processes.
-rule mpi-test ( name : sources * : requirements * : schedule * )
-{
- sources ?= $(name).cpp ;
- schedule ?= 1 2 3 4 7 8 13 17 ;
-
- local result ;
- for processes in $(schedule)
- {
- result += [ testing.make-test
- run-mpi : $(sources) /boost/mpi//boost_mpi
- : $(requirements) <toolset>msvc:<link>static <mpi:processes>$(processes) : $(name)-$(processes) ] ;
- }
- return $(result) ;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/msvc-config.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/msvc-config.jam b/ext/kenlm/jam-files/boost-build/tools/msvc-config.jam
deleted file mode 100644
index 6c71e3b..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/msvc-config.jam
+++ /dev/null
@@ -1,12 +0,0 @@
-#~ Copyright 2005 Rene Rivera.
-#~ Distributed under the Boost Software License, Version 1.0.
-#~ (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-# Automatic configuration for VisualStudio toolset. To use, just import this module.
-
-import toolset : using ;
-
-ECHO "warning: msvc-config.jam is deprecated. Use 'using msvc : all ;' instead." ;
-
-using msvc : all ;
-
[31/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/util/print.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/util/print.jam b/ext/kenlm/jam-files/boost-build/util/print.jam
deleted file mode 100644
index c867e4e..0000000
--- a/ext/kenlm/jam-files/boost-build/util/print.jam
+++ /dev/null
@@ -1,488 +0,0 @@
-# Copyright 2003 Douglas Gregor
-# Copyright 2002, 2003, 2005 Rene Rivera
-# Copyright 2002, 2003, 2004, 2005 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-# Utilities for generating format independent output. Using these
-# will help in generation of documentation in at minimum plain/console
-# and html.
-
-import modules ;
-import numbers ;
-import string ;
-import regex ;
-import "class" ;
-import scanner ;
-import path ;
-
-# The current output target. Defaults to console.
-output-target = console ;
-
-# The current output type. Defaults to plain. Other possible values are "html".
-output-type = plain ;
-
-# Whitespace.
-.whitespace = [ string.whitespace ] ;
-
-
-# Set the target and type of output to generate. This sets both the destination
-# output and the type of docs to generate to that output. The target can be
-# either a file or "console" for echoing to the console. If the type of output
-# is not specified it defaults to plain text.
-#
-rule output (
- target # The target file or device; file or "console".
- type ? # The type of output; "plain" or "html".
-)
-{
- type ?= plain ;
- if $(output-target) != $(target)
- {
- output-target = $(target) ;
- output-type = $(type) ;
- if $(output-type) = html
- {
- text
- "<!DOCTYPE html PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\">"
- "<html>"
- "<head>"
- "</head>"
- "<body link=\"#0000ff\" vlink=\"#800080\">"
- : true
- : prefix ;
- text
- "</body>"
- "</html>"
- :
- : suffix ;
- }
- }
-}
-
-
-# Generate a section with a description. The type of output can be controlled by
-# the value of the 'output-type' variable.
-#
-rule section (
- name # The name of the section.
- description * # A number of description lines.
-)
-{
- if $(output-type) = plain
- {
- lines [ split-at-words $(name): ] ;
- lines ;
- }
- else if $(output-type) = html
- {
- name = [ escape-html $(name) ] ;
- text <h3>$(name)</h3> <p> ;
- }
- local pre = ;
- while $(description)
- {
- local paragraph = ;
- while $(description) && [ string.is-whitespace $(description[1]) ] { description = $(description[2-]) ; }
- if $(pre)
- {
- while $(description) && (
- $(pre) = " $(description[1])" ||
- ( $(pre) < [ string.chars [ MATCH "^([$(.whitespace)]*)" : " $(description[1])" ] ] )
- )
- { paragraph += $(description[1]) ; description = $(description[2-]) ; }
- while [ string.is-whitespace $(paragraph[-1]) ] { paragraph = $(paragraph[1--2]) ; }
- pre = ;
- if $(output-type) = plain
- {
- lines $(paragraph) "" : " " " " ;
- }
- else if $(output-type) = html
- {
- text <blockquote> ;
- lines $(paragraph) ;
- text </blockquote> ;
- }
- }
- else
- {
- while $(description) && ! [ string.is-whitespace $(description[1]) ]
- { paragraph += $(description[1]) ; description = $(description[2-]) ; }
- if $(paragraph[1]) = :: && ! $(paragraph[2])
- {
- pre = " " ;
- }
- if $(paragraph[1]) = ::
- {
- if $(output-type) = plain
- {
- lines $(paragraph[2-]) "" : " " " " ;
- lines ;
- }
- else if $(output-type) = html
- {
- text <blockquote> ;
- lines $(paragraph[2-]) ;
- text </blockquote> ;
- }
- }
- else
- {
- local p = [ MATCH "(.*)(::)$" : $(paragraph[-1]) ] ;
- local pws = [ MATCH "([ ]*)$" : $(p[1]) ] ;
- p = [ MATCH "(.*)($(pws))($(p[2]))$" : $(paragraph[-1]) ] ;
- if $(p[3]) = ::
- {
- pre = [ string.chars [ MATCH "^([$(.whitespace)]*)" : " $(p[1])" ] ] ;
- if ! $(p[2]) || $(p[2]) = "" { paragraph = $(paragraph[1--2]) $(p[1]): ; }
- else { paragraph = $(paragraph[1--2]) $(p[1]) ; }
- if $(output-type) = plain
- {
- lines [ split-at-words " " $(paragraph) ] : " " " " ;
- lines ;
- }
- else if $(output-type) = html
- {
- text </p> <p> [ escape-html $(paragraph) ] ;
- }
- }
- else
- {
- if $(output-type) = plain
- {
- lines [ split-at-words " " $(paragraph) ] : " " " " ;
- lines ;
- }
- else if $(output-type) = html
- {
- text </p> <p> [ escape-html $(paragraph) ] ;
- }
- }
- }
- }
- }
- if $(output-type) = html
- {
- text </p> ;
- }
-}
-
-
-# Generate the start of a list of items. The type of output can be controlled by
-# the value of the 'output-type' variable.
-#
-rule list-start ( )
-{
- if $(output-type) = plain
- {
- }
- else if $(output-type) = html
- {
- text <ul> ;
- }
-}
-
-
-# Generate an item in a list. The type of output can be controlled by the value
-# of the 'output-type' variable.
-#
-rule list-item (
- item + # The item to list.
-)
-{
- if $(output-type) = plain
- {
- lines [ split-at-words "*" $(item) ] : " " " " ;
- }
- else if $(output-type) = html
- {
- text <li> [ escape-html $(item) ] </li> ;
- }
-}
-
-
-# Generate the end of a list of items. The type of output can be controlled by
-# the value of the 'output-type' variable.
-#
-rule list-end ( )
-{
- if $(output-type) = plain
- {
- lines ;
- }
- else if $(output-type) = html
- {
- text </ul> ;
- }
-}
-
-
-# Split the given text into separate lines, word-wrapping to a margin. The
-# default margin is 78 characters.
-#
-rule split-at-words (
- text + # The text to split.
- : margin ? # An optional margin, default is 78.
-)
-{
- local lines = ;
- text = [ string.words $(text:J=" ") ] ;
- text = $(text:J=" ") ;
- margin ?= 78 ;
- local char-match-1 = ".?" ;
- local char-match = "" ;
- while $(margin) != 0
- {
- char-match = $(char-match)$(char-match-1) ;
- margin = [ numbers.decrement $(margin) ] ;
- }
- while $(text)
- {
- local s = "" ;
- local t = "" ;
- # divide s into the first X characters and the rest
- s = [ MATCH "^($(char-match))(.*)" : $(text) ] ;
-
- if $(s[2])
- {
- # split the first half at a space
- t = [ MATCH "^(.*)[\\ ]([^\\ ]*)$" : $(s[1]) ] ;
- }
- else
- {
- t = $(s) ;
- }
-
- if ! $(t[2])
- {
- t += "" ;
- }
-
- text = $(t[2])$(s[2]) ;
- lines += $(t[1]) ;
- }
- return $(lines) ;
-}
-
-
-# Generate a set of fixed lines. Each single item passed in is output on a
-# separate line. For console this just echos each line, but for html this will
-# split them with <br>.
-#
-rule lines (
- text * # The lines of text.
- : indent ? # Optional indentation prepended to each line after the first.
- outdent ? # Optional indentation to prepend to the first line.
-)
-{
- text ?= "" ;
- indent ?= "" ;
- outdent ?= "" ;
- if $(output-type) = plain
- {
- text $(outdent)$(text[1]) $(indent)$(text[2-]) ;
- }
- else if $(output-type) = html
- {
- local indent-chars = [ string.chars $(indent) ] ;
- indent = "" ;
- for local c in $(indent-chars)
- {
- if $(c) = " " { c = " " ; }
- else if $(c) = " " { c = " " ; }
- indent = $(indent)$(c) ;
- }
- local html-text = [ escape-html $(text) : " " ] ;
- text $(html-text[1])<br> $(indent)$(html-text[2-])<br> ;
- }
-}
-
-
-# Output text directly to the current target. When doing output to a file, one
-# can indicate if the text should be output to "prefix" it, as the "body"
-# (default), or "suffix" of the file. This is independant of the actual
-# execution order of the text rule. This rule invokes a singular action, one
-# action only once, which does the build of the file. Therefore actions on the
-# target outside of this rule will happen entirely before and/or after all
-# output using this rule.
-#
-rule text (
- strings * # The strings of text to output.
- : overwrite ? # True to overwrite the output (if it is a file).
- : prefix-body-suffix ? # Indication to output prefix, body, or suffix (for
- # a file).
-)
-{
- prefix-body-suffix ?= body ;
- if $(output-target) = console
- {
- if ! $(strings)
- {
- ECHO ;
- }
- else
- {
- for local s in $(strings)
- {
- ECHO $(s) ;
- }
- }
- }
- if ! $($(output-target).did-action)
- {
- $(output-target).did-action = yes ;
- $(output-target).text-prefix = ;
- $(output-target).text-body = ;
- $(output-target).text-suffix = ;
-
- nl on $(output-target) = "
-" ;
- text-redirect on $(output-target) = ">>" ;
- if $(overwrite)
- {
- text-redirect on $(output-target) = ">" ;
- }
- text-content on $(output-target) = ;
-
- text-action $(output-target) ;
-
- if $(overwrite) && $(output-target) != console
- {
- check-for-update $(output-target) ;
- }
- }
- $(output-target).text-$(prefix-body-suffix) += $(strings) ;
- text-content on $(output-target) =
- $($(output-target).text-prefix)
- $($(output-target).text-body)
- $($(output-target).text-suffix) ;
-}
-
-
-# Outputs the text to the current targets, after word-wrapping it.
-#
-rule wrapped-text ( text + )
-{
- local lines = [ split-at-words $(text) ] ;
- text $(lines) ;
-}
-
-
-# Escapes text into html/xml printable equivalents. Does not know about tags and
-# therefore tags fed into this will also be escaped. Currently escapes space,
-# "<", ">", and "&".
-#
-rule escape-html (
- text + # The text to escape.
- : space ? # What to replace spaces with, defaults to " ".
-)
-{
- local html-text = ;
- while $(text)
- {
- local html = $(text[1]) ;
- text = $(text[2-]) ;
- html = [ regex.replace $(html) "&" "&" ] ;
- html = [ regex.replace $(html) "<" "<" ] ;
- html = [ regex.replace $(html) ">" ">" ] ;
- if $(space)
- {
- html = [ regex.replace $(html) " " "$(space)" ] ;
- }
- html-text += $(html) ;
- }
- return $(html-text) ;
-}
-
-
-# Outputs the text strings collected by the text rule to the output file.
-#
-actions quietly text-action
-{
- @($(STDOUT):E=$(text-content:J=$(nl))) $(text-redirect) "$(<)"
-}
-
-
-rule get-scanner ( )
-{
- if ! $(.scanner)
- {
- .scanner = [ class.new print-scanner ] ;
- }
- return $(.scanner) ;
-}
-
-
-# The following code to update print targets when their contents change is a
-# horrible hack. It basically creates a target which binds to this file
-# (print.jam) and installs a scanner on it which reads the target and compares
-# its contents to the new contents that we are writing.
-#
-rule check-for-update ( target )
-{
- local scanner = [ get-scanner ] ;
- local file = [ path.native [ modules.binding $(__name__) ] ] ;
- local g = [ MATCH <(.*)> : $(target:G) ] ;
- local dependency-target = $(__file__:G=$(g:E=)-$(target:G=)-$(scanner)) ;
- DEPENDS $(target) : $(dependency-target) ;
- SEARCH on $(dependency-target) = $(file:D) ;
- ISFILE $(dependency-target) ;
- NOUPDATE $(dependency-target) ;
- base on $(dependency-target) = $(target) ;
- scanner.install $(scanner) : $(dependency-target) ;
- return $(dependency-target) ;
-}
-
-
-class print-scanner : scanner
-{
- import path ;
- import os ;
-
- rule pattern ( )
- {
- return "(One match...)" ;
- }
-
- rule process ( target : matches * : binding )
- {
- local base = [ on $(target) return $(base) ] ;
- local nl = [ on $(base) return $(nl) ] ;
- local text-content = [ on $(base) return $(text-content) ] ;
- local dir = [ on $(base) return $(LOCATE) ] ;
- if $(dir)
- {
- dir = [ path.make $(dir) ] ;
- }
- local file = [ path.native [ path.join $(dir) $(base:G=) ] ] ;
- local actual-content ;
- if [ os.name ] = NT
- {
- actual-content = [ SHELL "type \"$(file)\" 2>nul" ] ;
- }
- else
- {
- actual-content = [ SHELL "cat \"$(file)\" 2>/dev/null" ] ;
- }
- if $(text-content:J=$(nl)) != $(actual-content)
- {
- ALWAYS $(base) ;
- }
- }
-}
-
-
-rule __test__ ( )
-{
- import assert ;
-
- assert.result one two three : split-at-words one two three : 5 ;
- assert.result "one two" three : split-at-words one two three : 8 ;
- assert.result "one two" three : split-at-words one two three : 9 ;
- assert.result "one two three" : split-at-words one two three ;
-
- # VP, 2004-12-03 The following test fails for some reason, so commenting it
- # out.
- #assert.result "one two three" "&<>" :
- # escape-html "one two three" "&<>" ;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/util/regex.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/util/regex.jam b/ext/kenlm/jam-files/boost-build/util/regex.jam
deleted file mode 100644
index be8b3cf..0000000
--- a/ext/kenlm/jam-files/boost-build/util/regex.jam
+++ /dev/null
@@ -1,203 +0,0 @@
-# Copyright 2001, 2002 Dave Abrahams
-# Copyright 2003 Douglas Gregor
-# Copyright 2003 Rene Rivera
-# Copyright 2002, 2003, 2004, 2005 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-#
-# Returns a list of the following substrings:
-# 1) from beginning till the first occurrence of 'separator' or till the end,
-# 2) between each occurrence of 'separator' and the next occurrence,
-# 3) from the last occurrence of 'separator' till the end.
-# If no separator is present, the result will contain only one element.
-#
-
-rule split ( string separator )
-{
- local result ;
- local s = $(string) ;
-
- # Break pieaces off 's' until it has no separators left.
- local match = 1 ;
- while $(match)
- {
- match = [ MATCH ^(.*)($(separator))(.*) : $(s) ] ;
- if $(match)
- {
- match += "" ; # in case 3rd item was empty - works around MATCH bug
- result = $(match[3]) $(result) ;
- s = $(match[1]) ;
- }
- }
- # Combine the remaining part at the beginning, which does not have
- # separators, with the pieces broken off. Note that the rule's signature
- # does not allow the initial s to be empty.
- return $(s) $(result) ;
-}
-
-if [ HAS_NATIVE_RULE regex : split : 1 ]
-{
- NATIVE_RULE regex : split ;
-}
-
-# Returns the concatenated results of Applying regex.split to every element of
-# the list using the separator pattern.
-#
-rule split-list ( list * : separator )
-{
- local result ;
- for s in $(list)
- {
- result += [ split $(s) $(separator) ] ;
- }
- return $(result) ;
-}
-
-
-# Match string against pattern, and return the elements indicated by indices.
-#
-rule match ( pattern : string : indices * )
-{
- indices ?= 1 2 3 4 5 6 7 8 9 ;
- local x = [ MATCH $(pattern) : $(string) ] ;
- return $(x[$(indices)]) ;
-}
-
-
-# Matches all elements of 'list' agains the 'pattern' and returns a list of
-# elements indicated by indices of all successful matches. If 'indices' is
-# omitted returns a list of first parenthesised groups of all successful
-# matches.
-#
-rule transform ( list * : pattern : indices * )
-{
- indices ?= 1 ;
- local result ;
- for local e in $(list)
- {
- local m = [ MATCH $(pattern) : $(e) ] ;
- if $(m)
- {
- result += $(m[$(indices)]) ;
- }
- }
- return $(result) ;
-}
-
-NATIVE_RULE regex : transform ;
-
-
-# Escapes all of the characters in symbols using the escape symbol escape-symbol
-# for the given string, and returns the escaped string.
-#
-rule escape ( string : symbols : escape-symbol )
-{
- local result = "" ;
- local m = 1 ;
- while $(m)
- {
- m = [ MATCH ^([^$(symbols)]*)([$(symbols)])(.*) : $(string) ] ;
- if $(m)
- {
- m += "" ; # Supposedly a bug fix; borrowed from regex.split
- result = "$(result)$(m[1])$(escape-symbol)$(m[2])" ;
- string = $(m[3]) ;
- }
- }
- string ?= "" ;
- result = "$(result)$(string)" ;
- return $(result) ;
-}
-
-
-# Replaces occurrences of a match string in a given string and returns the new
-# string. The match string can be a regex expression.
-#
-rule replace (
- string # The string to modify.
- match # The characters to replace.
- replacement # The string to replace with.
- )
-{
- local result = "" ;
- local parts = 1 ;
- while $(parts)
- {
- parts = [ MATCH ^(.*)($(match))(.*) : $(string) ] ;
- if $(parts)
- {
- parts += "" ;
- result = "$(replacement)$(parts[3])$(result)" ;
- string = $(parts[1]) ;
- }
- }
- string ?= "" ;
- result = "$(string)$(result)" ;
- return $(result) ;
-}
-
-if [ HAS_NATIVE_RULE regex : replace : 1 ]
-{
- NATIVE_RULE regex : replace ;
-}
-
-
-# Replaces occurrences of a match string in a given list of strings and returns
-# a list of new strings. The match string can be a regex expression.
-#
-# list - the list of strings to modify.
-# match - the search expression.
-# replacement - the string to replace with.
-#
-rule replace-list ( list * : match : replacement )
-{
- local result ;
- for local e in $(list)
- {
- result += [ replace $(e) $(match) $(replacement) ] ;
- }
- return $(result) ;
-}
-
-
-rule __test__ ( )
-{
- import assert ;
-
- assert.result a b c : split "a/b/c" / ;
- assert.result "" a b c : split "/a/b/c" / ;
- assert.result "" "" a b c : split "//a/b/c" / ;
- assert.result "" a "" b c : split "/a//b/c" / ;
- assert.result "" a "" b c "" : split "/a//b/c/" / ;
- assert.result "" a "" b c "" "" : split "/a//b/c//" / ;
-
- assert.result a c b d
- : match (.)(.)(.)(.) : abcd : 1 3 2 4 ;
-
- assert.result a b c d
- : match (.)(.)(.)(.) : abcd ;
-
- assert.result ababab cddc
- : match ((ab)*)([cd]+) : abababcddc : 1 3 ;
-
- assert.result a.h c.h
- : transform <a.h> \"b.h\" <c.h> : <(.*)> ;
-
- assert.result a.h b.h c.h
- : transform <a.h> \"b.h\" <c.h> : <([^>]*)>|\"([^\"]*)\" : 1 2 ;
-
- assert.result "^<?xml version=\"1.0\"^>"
- : escape "<?xml version=\"1.0\">" : "&|()<>^" : "^" ;
-
- assert.result "<?xml version=\\\"1.0\\\">"
- : escape "<?xml version=\"1.0\">" : "\\\"" : "\\" ;
-
- assert.result "string string " : replace "string string " " " " " ;
- assert.result " string string" : replace " string string" " " " " ;
- assert.result "string string" : replace "string string" " " " " ;
- assert.result "-" : replace "&" "&" "-" ;
-
- assert.result "-" "a-b" : replace-list "&" "a&b" : "&" : "-" ;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/util/sequence.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/util/sequence.jam b/ext/kenlm/jam-files/boost-build/util/sequence.jam
deleted file mode 100644
index a87eb4d..0000000
--- a/ext/kenlm/jam-files/boost-build/util/sequence.jam
+++ /dev/null
@@ -1,339 +0,0 @@
-# Copyright 2001, 2002, 2003 Dave Abrahams
-# Copyright 2006 Rene Rivera
-# Copyright 2002, 2003 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-import assert ;
-import numbers ;
-import modules ;
-
-
-# Note that algorithms in this module execute largely in the caller's module
-# namespace, so that local rules can be used as function objects. Also note that
-# most predicates can be multi-element lists. In that case, all but the first
-# element are prepended to the first argument which is passed to the rule named
-# by the first element.
-
-
-# Return the elements e of $(sequence) for which [ $(predicate) e ] has a
-# non-null value.
-#
-rule filter ( predicate + : sequence * )
-{
- local caller = [ CALLER_MODULE ] ;
- local result ;
-
- for local e in $(sequence)
- {
- if [ modules.call-in $(caller) : $(predicate) $(e) ]
- {
- result += $(e) ;
- }
- }
- return $(result) ;
-}
-
-
-# Return a new sequence consisting of [ $(function) $(e) ] for each element e of
-# $(sequence).
-#
-rule transform ( function + : sequence * )
-{
- local caller = [ CALLER_MODULE ] ;
- local result ;
-
- for local e in $(sequence)
- {
- result += [ modules.call-in $(caller) : $(function) $(e) ] ;
- }
- return $(result) ;
-}
-
-if [ HAS_NATIVE_RULE sequence : transform : 1 ]
-{
- NATIVE_RULE sequence : transform ;
-}
-
-rule reverse ( s * )
-{
- local r ;
- for local x in $(s)
- {
- r = $(x) $(r) ;
- }
- return $(r) ;
-}
-
-
-rule less ( a b )
-{
- if $(a) < $(b)
- {
- return true ;
- }
-}
-
-
-# Insertion-sort s using the BinaryPredicate ordered.
-#
-rule insertion-sort ( s * : ordered * )
-{
- if ! $(ordered)
- {
- return [ SORT $(s) ] ;
- }
- else
- {
- local caller = [ CALLER_MODULE ] ;
- ordered ?= sequence.less ;
- local result = $(s[1]) ;
- if $(ordered) = sequence.less
- {
- local head tail ;
- for local x in $(s[2-])
- {
- head = ;
- tail = $(result) ;
- while $(tail) && ( $(tail[1]) < $(x) )
- {
- head += $(tail[1]) ;
- tail = $(tail[2-]) ;
- }
- result = $(head) $(x) $(tail) ;
- }
- }
- else
- {
- for local x in $(s[2-])
- {
- local head tail ;
- tail = $(result) ;
- while $(tail) && [ modules.call-in $(caller) : $(ordered) $(tail[1]) $(x) ]
- {
- head += $(tail[1]) ;
- tail = $(tail[2-]) ;
- }
- result = $(head) $(x) $(tail) ;
- }
- }
-
- return $(result) ;
- }
-}
-
-
-# Merge two ordered sequences using the BinaryPredicate ordered.
-#
-rule merge ( s1 * : s2 * : ordered * )
-{
- ordered ?= sequence.less ;
- local result__ ;
- local caller = [ CALLER_MODULE ] ;
-
- while $(s1) && $(s2)
- {
- if [ modules.call-in $(caller) : $(ordered) $(s1[1]) $(s2[1]) ]
- {
- result__ += $(s1[1]) ;
- s1 = $(s1[2-]) ;
- }
- else if [ modules.call-in $(caller) : $(ordered) $(s2[1]) $(s1[1]) ]
- {
- result__ += $(s2[1]) ;
- s2 = $(s2[2-]) ;
- }
- else
- {
- s2 = $(s2[2-]) ;
- }
-
- }
- result__ += $(s1) ;
- result__ += $(s2) ;
-
- return $(result__) ;
-}
-
-
-# Join the elements of s into one long string. If joint is supplied, it is used
-# as a separator.
-#
-rule join ( s * : joint ? )
-{
- joint ?= "" ;
- return $(s:J=$(joint)) ;
-}
-
-
-# Find the length of any sequence.
-#
-rule length ( s * )
-{
- local result = 0 ;
- for local i in $(s)
- {
- result = [ CALC $(result) + 1 ] ;
- }
- return $(result) ;
-}
-
-
-rule unique ( list * : stable ? )
-{
- local result ;
- local prev ;
- if $(stable)
- {
- for local f in $(list)
- {
- if ! $(f) in $(result)
- {
- result += $(f) ;
- }
- }
- }
- else
- {
- for local i in [ SORT $(list) ]
- {
- if $(i) != $(prev)
- {
- result += $(i) ;
- }
- prev = $(i) ;
- }
- }
- return $(result) ;
-}
-
-
-# Returns the maximum number in 'elements'. Uses 'ordered' for comparisons or
-# 'numbers.less' if none is provided.
-#
-rule max-element ( elements + : ordered ? )
-{
- ordered ?= numbers.less ;
-
- local max = $(elements[1]) ;
- for local e in $(elements[2-])
- {
- if [ $(ordered) $(max) $(e) ]
- {
- max = $(e) ;
- }
- }
- return $(max) ;
-}
-
-
-# Returns all of 'elements' for which corresponding element in parallel list
-# 'rank' is equal to the maximum value in 'rank'.
-#
-rule select-highest-ranked ( elements * : ranks * )
-{
- if $(elements)
- {
- local max-rank = [ max-element $(ranks) ] ;
- local result ;
- while $(elements)
- {
- if $(ranks[1]) = $(max-rank)
- {
- result += $(elements[1]) ;
- }
- elements = $(elements[2-]) ;
- ranks = $(ranks[2-]) ;
- }
- return $(result) ;
- }
-}
-NATIVE_RULE sequence : select-highest-ranked ;
-
-
-rule __test__ ( )
-{
- # Use a unique module so we can test the use of local rules.
- module sequence.__test__
- {
- import assert ;
- import sequence ;
-
- local rule is-even ( n )
- {
- if $(n) in 0 2 4 6 8
- {
- return true ;
- }
- }
-
- assert.result 4 6 4 2 8 : sequence.filter is-even : 1 4 6 3 4 7 2 3 8 ;
-
- # Test that argument binding works.
- local rule is-equal-test ( x y )
- {
- if $(x) = $(y)
- {
- return true ;
- }
- }
-
- assert.result 3 3 3 : sequence.filter is-equal-test 3 : 1 2 3 4 3 5 3 5 7 ;
-
- local rule append-x ( n )
- {
- return $(n)x ;
- }
-
- assert.result 1x 2x 3x : sequence.transform append-x : 1 2 3 ;
-
- local rule repeat2 ( x )
- {
- return $(x) $(x) ;
- }
-
- assert.result 1 1 2 2 3 3 : sequence.transform repeat2 : 1 2 3 ;
-
- local rule test-greater ( a b )
- {
- if $(a) > $(b)
- {
- return true ;
- }
- }
- assert.result 1 2 3 4 5 6 7 8 9 : sequence.insertion-sort 9 6 5 3 8 7 1 2 4 ;
- assert.result 9 8 7 6 5 4 3 2 1 : sequence.insertion-sort 9 6 5 3 8 7 1 2 4 : test-greater ;
- assert.result 1 2 3 4 5 6 : sequence.merge 1 3 5 : 2 4 6 ;
- assert.result 6 5 4 3 2 1 : sequence.merge 5 3 1 : 6 4 2 : test-greater ;
- assert.result 1 2 3 : sequence.merge 1 2 3 : ;
- assert.result 1 : sequence.merge 1 : 1 ;
-
- assert.result foo-bar-baz : sequence.join foo bar baz : - ;
- assert.result substandard : sequence.join sub stan dard ;
- assert.result 3.0.1 : sequence.join 3.0.1 : - ;
-
- assert.result 0 : sequence.length ;
- assert.result 3 : sequence.length a b c ;
- assert.result 17 : sequence.length 17 16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 ;
-
- assert.result 1 : sequence.length a ;
- assert.result 10 : sequence.length a b c d e f g h i j ;
- assert.result 11 : sequence.length a b c d e f g h i j k ;
- assert.result 12 : sequence.length a b c d e f g h i j k l ;
-
- local p2 = x ;
- for local i in 1 2 3 4 5 6 7 8
- {
- p2 = $(p2) $(p2) ;
- }
- assert.result 256 : sequence.length $(p2) ;
-
- assert.result 1 2 3 4 5 : sequence.unique 1 2 3 2 4 3 3 5 5 5 ;
-
- assert.result 5 : sequence.max-element 1 3 5 0 4 ;
-
- assert.result e-3 h-3 : sequence.select-highest-ranked e-1 e-3 h-3 m-2 : 1 3 3 2 ;
-
- assert.result 7 6 5 4 3 2 1 : sequence.reverse 1 2 3 4 5 6 7 ;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/util/set.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/util/set.jam b/ext/kenlm/jam-files/boost-build/util/set.jam
deleted file mode 100644
index fc17913..0000000
--- a/ext/kenlm/jam-files/boost-build/util/set.jam
+++ /dev/null
@@ -1,93 +0,0 @@
-# Copyright 2001, 2002 Dave Abrahams
-# Copyright 2003 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-class set
-{
- rule __init__ ( )
- {
- }
-
- rule add ( elements * )
- {
- for local e in $(elements)
- {
- if ! $($(e))
- {
- $(e) = 1 ;
- self.result += $(e) ;
- }
- }
- }
-
- rule contains ( element )
- {
- return $($(element)) ;
- }
-
- rule list ( )
- {
- return $(self.result) ;
- }
-}
-
-
-
-# Returns the elements of set1 that are not in set2.
-#
-rule difference ( set1 * : set2 * )
-{
- local result = ;
- for local element in $(set1)
- {
- if ! ( $(element) in $(set2) )
- {
- result += $(element) ;
- }
- }
- return $(result) ;
-}
-
-NATIVE_RULE set : difference ;
-
-
-# Removes all the items appearing in both set1 & set2.
-#
-rule intersection ( set1 * : set2 * )
-{
- local result ;
- for local v in $(set1)
- {
- if $(v) in $(set2)
- {
- result += $(v) ;
- }
- }
- return $(result) ;
-}
-
-
-# Returns whether set1 & set2 contain the same elements. Note that this ignores
-# any element ordering differences as well as any element duplication.
-#
-rule equal ( set1 * : set2 * )
-{
- if $(set1) in $(set2) && ( $(set2) in $(set1) )
- {
- return true ;
- }
-}
-
-
-rule __test__ ( )
-{
- import assert ;
-
- assert.result 0 1 4 6 8 9 : difference 0 1 2 3 4 5 6 7 8 9 : 2 3 5 7 ;
- assert.result 2 5 7 : intersection 0 1 2 4 5 6 7 8 9 : 2 3 5 7 ;
-
- assert.true equal : ;
- assert.true equal 1 1 2 3 : 3 2 2 1 ;
- assert.false equal 2 3 : 3 2 2 1 ;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/util/string.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/util/string.jam b/ext/kenlm/jam-files/boost-build/util/string.jam
deleted file mode 100644
index a39ed11..0000000
--- a/ext/kenlm/jam-files/boost-build/util/string.jam
+++ /dev/null
@@ -1,189 +0,0 @@
-# Copyright 2002 Dave Abrahams
-# Copyright 2002, 2003 Rene Rivera
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-import regex ;
-
-
-# Characters considered whitespace, as a list.
-.whitespace-chars = " " " " "
-" ;
-
-# Characters considered whitespace, as a single string.
-.whitespace = $(.whitespace-chars:J="") ;
-
-
-# Returns the canonical set of whitespace characters, as a list.
-#
-rule whitespace-chars ( )
-{
- return $(.whitespace-chars) ;
-}
-
-
-# Returns the canonical set of whitespace characters, as a single string.
-#
-rule whitespace ( )
-{
- return $(.whitespace) ;
-}
-
-
-# Splits the given string into a list of strings composed of each character of
-# the string in sequence.
-#
-rule chars (
- string # The string to split.
- )
-{
- local result ;
- while $(string)
- {
- local s = [ MATCH (.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.*) : $(string) ] ;
- string = $(s[9]) ;
- result += $(s[1-8]) ;
- }
-
- # Trim off empty strings.
- while $(result[1]) && ! $(result[-1])
- {
- result = $(result[1--2]) ;
- }
-
- return $(result) ;
-}
-
-
-# Apply a set of standard transformations to string to produce an abbreviation
-# no more than 5 characters long.
-#
-rule abbreviate ( string )
-{
- local r = $(.abbreviated-$(string)) ;
- if $(r)
- {
- return $(r) ;
- }
- # Anything less than 4 characters gets no abbreviation.
- else if ! [ MATCH (....) : $(string) ]
- {
- .abbreviated-$(string) = $(string) ;
- return $(string) ;
- }
- else
- {
- # Separate the initial letter in case it's a vowel.
- local s1 = [ MATCH ^(.)(.*) : $(string) ] ;
-
- # Drop trailing "ing".
- local s2 = [ MATCH ^(.*)ing$ : $(s1[2]) ] ;
- s2 ?= $(s1[2]) ;
-
- # Reduce all doubled characters to one.
- local last = "" ;
- for local c in [ chars $(s2) ]
- {
- if $(c) != $(last)
- {
- r += $(c) ;
- last = $(c) ;
- }
- }
- s2 = $(r:J="") ;
-
- # Chop all vowels out of the remainder.
- s2 = [ regex.replace $(s2) [AEIOUaeiou] "" ] ;
-
- # Shorten remaining consonants to 4 characters.
- s2 = [ MATCH ^(.?.?.?.?) : $(s2) ] ;
-
- # Glue the initial character back on to the front.
- s2 = $(s1[1])$(s2) ;
-
- .abbreviated-$(string) = $(s2) ;
- return $(s2) ;
- }
-}
-
-
-# Concatenates the given strings, inserting the given separator between each
-# string.
-#
-rule join (
- strings * # The strings to join.
- : separator ? # The optional separator.
- )
-{
- separator ?= "" ;
- return $(strings:J=$(separator)) ;
-}
-
-
-# Split a string into whitespace separated words.
-#
-rule words (
- string # The string to split.
- : whitespace * # Optional, characters to consider as whitespace.
- )
-{
- whitespace = $(whitespace:J="") ;
- whitespace ?= $(.whitespace) ;
- local w = ;
- while $(string)
- {
- string = [ MATCH "^[$(whitespace)]*([^$(whitespace)]*)(.*)" : $(string) ] ;
- if $(string[1]) && $(string[1]) != ""
- {
- w += $(string[1]) ;
- }
- string = $(string[2]) ;
- }
- return $(w) ;
-}
-
-
-# Check that the given string is composed entirely of whitespace.
-#
-rule is-whitespace (
- string ? # The string to test.
- )
-{
- if ! $(string) { return true ; }
- else if $(string) = "" { return true ; }
- else if [ MATCH "^([$(.whitespace)]+)$" : $(string) ] { return true ; }
- else { return ; }
-}
-
-rule __test__ ( )
-{
- import assert ;
- assert.result a b c : chars abc ;
-
- assert.result rntm : abbreviate runtime ;
- assert.result ovrld : abbreviate overload ;
- assert.result dbg : abbreviate debugging ;
- assert.result async : abbreviate asynchronous ;
- assert.result pop : abbreviate pop ;
- assert.result aaa : abbreviate aaa ;
- assert.result qck : abbreviate quack ;
- assert.result sttc : abbreviate static ;
-
- # Check boundary cases.
- assert.result a : chars a ;
- assert.result : chars "" ;
- assert.result a b c d e f g h : chars abcdefgh ;
- assert.result a b c d e f g h i : chars abcdefghi ;
- assert.result a b c d e f g h i j : chars abcdefghij ;
- assert.result a b c d e f g h i j k : chars abcdefghijk ;
-
- assert.result a//b/c/d : join a "" b c d : / ;
- assert.result abcd : join a "" b c d ;
-
- assert.result a b c : words "a b c" ;
-
- assert.true is-whitespace " " ;
- assert.false is-whitespace " a b c " ;
- assert.true is-whitespace "" ;
- assert.true is-whitespace ;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/util/utility.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/util/utility.jam b/ext/kenlm/jam-files/boost-build/util/utility.jam
deleted file mode 100644
index 26981c5..0000000
--- a/ext/kenlm/jam-files/boost-build/util/utility.jam
+++ /dev/null
@@ -1,235 +0,0 @@
-# Copyright 2001, 2002 Dave Abrahams
-# Copyright 2002, 2003, 2004, 2005 Vladimir Prus
-# Copyright 2008 Jurko Gospodnetic
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-import "class" : is-instance ;
-
-
-# For all elements of 'list' which do not already have 'suffix', add 'suffix'.
-#
-rule apply-default-suffix ( suffix : list * )
-{
- local result ;
- for local i in $(list)
- {
- if $(i:S) = $(suffix)
- {
- result += $(i) ;
- }
- else
- {
- result += $(i)$(suffix) ;
- }
- }
- return $(result) ;
-}
-
-
-# If 'name' contains a dot, returns the part before the last dot. If 'name'
-# contains no dot, returns it unmodified.
-#
-rule basename ( name )
-{
- if $(name:S)
- {
- name = $(name:B) ;
- }
- return $(name) ;
-}
-
-
-# Return the file of the caller of the rule that called caller-file.
-#
-rule caller-file ( )
-{
- local bt = [ BACKTRACE ] ;
- return $(bt[9]) ;
-}
-
-
-# Tests if 'a' is equal to 'b'. If 'a' is a class instance, calls its 'equal'
-# method. Uses ordinary jam's comparison otherwise.
-#
-rule equal ( a b )
-{
- if [ is-instance $(a) ]
- {
- return [ $(a).equal $(b) ] ;
- }
- else
- {
- if $(a) = $(b)
- {
- return true ;
- }
- }
-}
-
-
-# Tests if 'a' is less than 'b'. If 'a' is a class instance, calls its 'less'
-# method. Uses ordinary jam's comparison otherwise.
-#
-rule less ( a b )
-{
- if [ is-instance $(a) ]
- {
- return [ $(a).less $(b) ] ;
- }
- else
- {
- if $(a) < $(b)
- {
- return true ;
- }
- }
-}
-
-
-# Returns the textual representation of argument. If it is a class instance,
-# class its 'str' method. Otherwise, returns the argument.
-#
-rule str ( value )
-{
- if [ is-instance $(value) ]
- {
- return [ $(value).str ] ;
- }
- else
- {
- return $(value) ;
- }
-}
-
-
-# Accepts a list of gristed values and returns them ungristed. Reports an error
-# in case any of the passed parameters is not gristed, i.e. surrounded in angle
-# brackets < and >.
-#
-rule ungrist ( names * )
-{
- local result ;
- for local name in $(names)
- {
- local stripped = [ MATCH ^<(.*)>$ : $(name) ] ;
- if ! $(stripped)-defined
- {
- import errors ;
- local quoted-names = \"$(names)\" ;
- errors.error "in" ungrist $(quoted-names:J=" "): \"$(name)\" is not
- of the form <.*> ;
- }
- result += $(stripped) ;
- }
- return $(result) ;
-}
-
-
-# If the passed value is quoted, unquotes it. Otherwise returns the value
-# unchanged.
-#
-rule unquote ( value ? )
-{
- local match-result = [ MATCH ^(\")(.*)(\")$ : $(value) ] ;
- if $(match-result)
- {
- return $(match-result[2]) ;
- }
- else
- {
- return $(value) ;
- }
-}
-
-
-rule __test__ ( )
-{
- import assert ;
- import "class" : new ;
- import errors : try catch ;
-
- assert.result 123 : str 123 ;
-
- class test-class__
- {
- rule __init__ ( ) { }
- rule str ( ) { return "str-test-class" ; }
- rule less ( a ) { return "yes, of course!" ; }
- rule equal ( a ) { return "not sure" ; }
- }
-
- assert.result "str-test-class" : str [ new test-class__ ] ;
- assert.true less 1 2 ;
- assert.false less 2 1 ;
- assert.result "yes, of course!" : less [ new test-class__ ] 1 ;
- assert.true equal 1 1 ;
- assert.false equal 1 2 ;
- assert.result "not sure" : equal [ new test-class__ ] 1 ;
-
- assert.result foo.lib foo.lib : apply-default-suffix .lib : foo.lib foo.lib
- ;
-
- assert.result foo : basename foo ;
- assert.result foo : basename foo.so ;
- assert.result foo.so : basename foo.so.1 ;
-
- assert.result : unquote ;
- assert.result "" : unquote "" ;
- assert.result "" : unquote \"\" ;
- assert.result \" : unquote \"\"\" ;
- assert.result \"\" : unquote \"\"\"\" ;
- assert.result foo : unquote foo ;
- assert.result \"foo : unquote \"foo ;
- assert.result foo\" : unquote foo\" ;
- assert.result foo : unquote \"foo\" ;
- assert.result \"foo\" : unquote \"\"foo\"\" ;
-
- assert.result : ungrist ;
- assert.result "" : ungrist <> ;
- assert.result foo : ungrist <foo> ;
- assert.result <foo> : ungrist <<foo>> ;
- assert.result foo bar : ungrist <foo> <bar> ;
-
- try ;
- {
- ungrist "" ;
- }
- catch "in" ungrist \"\": \"\" is not of the form <.*> ;
-
- try ;
- {
- ungrist foo ;
- }
- catch "in" ungrist \"foo\": \"foo\" is not of the form <.*> ;
-
- try ;
- {
- ungrist <foo ;
- }
- catch "in" ungrist \"<foo\": \"<foo\" is not of the form <.*> ;
-
- try ;
- {
- ungrist foo> ;
- }
- catch "in" ungrist \"foo>\": \"foo>\" is not of the form <.*> ;
-
- try ;
- {
- ungrist foo bar ;
- }
- catch "in" ungrist "\"foo\" \"bar\"": \"foo\" is not of the form <.*> ;
-
- try ;
- {
- ungrist foo <bar> ;
- }
- catch "in" ungrist "\"foo\" \"<bar>\"": \"foo\" is not of the form <.*> ;
-
- try ;
- {
- ungrist <foo> bar ;
- }
- catch "in" ungrist "\"<foo>\" \"bar\"": \"bar\" is not of the form <.*> ;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/empty_test_main.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/empty_test_main.cc b/ext/kenlm/jam-files/empty_test_main.cc
deleted file mode 100644
index a3971f8..0000000
--- a/ext/kenlm/jam-files/empty_test_main.cc
+++ /dev/null
@@ -1,8 +0,0 @@
-/* Program to verify compilation against the unit test framework */
-
-#define BOOST_TEST_MODULE EmptyTest
-#include <boost/test/unit_test.hpp>
-
-namespace {
-BOOST_AUTO_TEST_CASE(Empty) {}
-} // namespace
[48/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/Doxyfile
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/Doxyfile b/ext/kenlm/Doxyfile
deleted file mode 100644
index 3abab65..0000000
--- a/ext/kenlm/Doxyfile
+++ /dev/null
@@ -1,1519 +0,0 @@
-# Doxyfile 1.6.1
-
-# This file describes the settings to be used by the documentation system
-# doxygen (www.doxygen.org) for a project
-#
-# All text after a hash (#) is considered a comment and will be ignored
-# The format is:
-# TAG = value [value, ...]
-# For lists items can also be appended using:
-# TAG += value [value, ...]
-# Values that contain spaces should be placed between quotes (" ")
-
-#---------------------------------------------------------------------------
-# Project related configuration options
-#---------------------------------------------------------------------------
-
-# This tag specifies the encoding used for all characters in the config file
-# that follow. The default is UTF-8 which is also the encoding used for all
-# text before the first occurrence of this tag. Doxygen uses libiconv (or the
-# iconv built into libc) for the transcoding. See
-# http://www.gnu.org/software/libiconv for the list of possible encodings.
-
-DOXYFILE_ENCODING = UTF-8
-
-# The PROJECT_NAME tag is a single word (or a sequence of words surrounded
-# by quotes) that should identify the project.
-
-PROJECT_NAME = KenLM
-
-# The PROJECT_NUMBER tag can be used to enter a project or revision number.
-# This could be handy for archiving the generated documentation or
-# if some version control system is used.
-
-PROJECT_NUMBER =
-
-# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute)
-# base path where the generated documentation will be put.
-# If a relative path is entered, it will be relative to the location
-# where doxygen was started. If left blank the current directory will be used.
-
-OUTPUT_DIRECTORY = doc
-
-# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create
-# 4096 sub-directories (in 2 levels) under the output directory of each output
-# format and will distribute the generated files over these directories.
-# Enabling this option can be useful when feeding doxygen a huge amount of
-# source files, where putting all generated files in the same directory would
-# otherwise cause performance problems for the file system.
-
-CREATE_SUBDIRS = NO
-
-# The OUTPUT_LANGUAGE tag is used to specify the language in which all
-# documentation generated by doxygen is written. Doxygen will use this
-# information to generate all constant output in the proper language.
-# The default language is English, other supported languages are:
-# Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional,
-# Croatian, Czech, Danish, Dutch, Esperanto, Farsi, Finnish, French, German,
-# Greek, Hungarian, Italian, Japanese, Japanese-en (Japanese with English
-# messages), Korean, Korean-en, Lithuanian, Norwegian, Macedonian, Persian,
-# Polish, Portuguese, Romanian, Russian, Serbian, Serbian-Cyrilic, Slovak,
-# Slovene, Spanish, Swedish, Ukrainian, and Vietnamese.
-
-OUTPUT_LANGUAGE = English
-
-# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will
-# include brief member descriptions after the members that are listed in
-# the file and class documentation (similar to JavaDoc).
-# Set to NO to disable this.
-
-BRIEF_MEMBER_DESC = YES
-
-# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend
-# the brief description of a member or function before the detailed description.
-# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the
-# brief descriptions will be completely suppressed.
-
-REPEAT_BRIEF = YES
-
-# This tag implements a quasi-intelligent brief description abbreviator
-# that is used to form the text in various listings. Each string
-# in this list, if found as the leading text of the brief description, will be
-# stripped from the text and the result after processing the whole list, is
-# used as the annotated text. Otherwise, the brief description is used as-is.
-# If left blank, the following values are used ("$name" is automatically
-# replaced with the name of the entity): "The $name class" "The $name widget"
-# "The $name file" "is" "provides" "specifies" "contains"
-# "represents" "a" "an" "the"
-
-ABBREVIATE_BRIEF =
-
-# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then
-# Doxygen will generate a detailed section even if there is only a brief
-# description.
-
-ALWAYS_DETAILED_SEC = YES
-
-# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all
-# inherited members of a class in the documentation of that class as if those
-# members were ordinary class members. Constructors, destructors and assignment
-# operators of the base classes will not be shown.
-
-INLINE_INHERITED_MEMB = NO
-
-# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full
-# path before files name in the file list and in the header files. If set
-# to NO the shortest path that makes the file name unique will be used.
-
-FULL_PATH_NAMES = YES
-
-# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag
-# can be used to strip a user-defined part of the path. Stripping is
-# only done if one of the specified strings matches the left-hand part of
-# the path. The tag can be used to show relative paths in the file list.
-# If left blank the directory from which doxygen is run is used as the
-# path to strip.
-
-STRIP_FROM_PATH =
-
-# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of
-# the path mentioned in the documentation of a class, which tells
-# the reader which header file to include in order to use a class.
-# If left blank only the name of the header file containing the class
-# definition is used. Otherwise one should specify the include paths that
-# are normally passed to the compiler using the -I flag.
-
-STRIP_FROM_INC_PATH =
-
-# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter
-# (but less readable) file names. This can be useful is your file systems
-# doesn't support long names like on DOS, Mac, or CD-ROM.
-
-SHORT_NAMES = NO
-
-# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen
-# will interpret the first line (until the first dot) of a JavaDoc-style
-# comment as the brief description. If set to NO, the JavaDoc
-# comments will behave just like regular Qt-style comments
-# (thus requiring an explicit @brief command for a brief description.)
-
-JAVADOC_AUTOBRIEF = YES
-
-# If the QT_AUTOBRIEF tag is set to YES then Doxygen will
-# interpret the first line (until the first dot) of a Qt-style
-# comment as the brief description. If set to NO, the comments
-# will behave just like regular Qt-style comments (thus requiring
-# an explicit \brief command for a brief description.)
-
-QT_AUTOBRIEF = NO
-
-# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen
-# treat a multi-line C++ special comment block (i.e. a block of //! or ///
-# comments) as a brief description. This used to be the default behaviour.
-# The new default is to treat a multi-line C++ comment block as a detailed
-# description. Set this tag to YES if you prefer the old behaviour instead.
-
-MULTILINE_CPP_IS_BRIEF = NO
-
-# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented
-# member inherits the documentation from any documented member that it
-# re-implements.
-
-INHERIT_DOCS = YES
-
-# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce
-# a new page for each member. If set to NO, the documentation of a member will
-# be part of the file/class/namespace that contains it.
-
-SEPARATE_MEMBER_PAGES = NO
-
-# The TAB_SIZE tag can be used to set the number of spaces in a tab.
-# Doxygen uses this value to replace tabs by spaces in code fragments.
-
-TAB_SIZE = 8
-
-# This tag can be used to specify a number of aliases that acts
-# as commands in the documentation. An alias has the form "name=value".
-# For example adding "sideeffect=\par Side Effects:\n" will allow you to
-# put the command \sideeffect (or @sideeffect) in the documentation, which
-# will result in a user-defined paragraph with heading "Side Effects:".
-# You can put \n's in the value part of an alias to insert newlines.
-
-ALIASES =
-
-# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C
-# sources only. Doxygen will then generate output that is more tailored for C.
-# For instance, some of the names that are used will be different. The list
-# of all members will be omitted, etc.
-
-OPTIMIZE_OUTPUT_FOR_C = NO
-
-# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java
-# sources only. Doxygen will then generate output that is more tailored for
-# Java. For instance, namespaces will be presented as packages, qualified
-# scopes will look different, etc.
-
-OPTIMIZE_OUTPUT_JAVA = NO
-
-# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran
-# sources only. Doxygen will then generate output that is more tailored for
-# Fortran.
-
-OPTIMIZE_FOR_FORTRAN = NO
-
-# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL
-# sources. Doxygen will then generate output that is tailored for
-# VHDL.
-
-OPTIMIZE_OUTPUT_VHDL = NO
-
-# Doxygen selects the parser to use depending on the extension of the files it parses.
-# With this tag you can assign which parser to use for a given extension.
-# Doxygen has a built-in mapping, but you can override or extend it using this tag.
-# The format is ext=language, where ext is a file extension, and language is one of
-# the parsers supported by doxygen: IDL, Java, Javascript, C#, C, C++, D, PHP,
-# Objective-C, Python, Fortran, VHDL, C, C++. For instance to make doxygen treat
-# .inc files as Fortran files (default is PHP), and .f files as C (default is Fortran),
-# use: inc=Fortran f=C. Note that for custom extensions you also need to set FILE_PATTERNS otherwise the files are not read by doxygen.
-
-EXTENSION_MAPPING =
-
-# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want
-# to include (a tag file for) the STL sources as input, then you should
-# set this tag to YES in order to let doxygen match functions declarations and
-# definitions whose arguments contain STL classes (e.g. func(std::string); v.s.
-# func(std::string) {}). This also make the inheritance and collaboration
-# diagrams that involve STL classes more complete and accurate.
-
-BUILTIN_STL_SUPPORT = NO
-
-# If you use Microsoft's C++/CLI language, you should set this option to YES to
-# enable parsing support.
-
-CPP_CLI_SUPPORT = NO
-
-# Set the SIP_SUPPORT tag to YES if your project consists of sip sources only.
-# Doxygen will parse them like normal C++ but will assume all classes use public
-# instead of private inheritance when no explicit protection keyword is present.
-
-SIP_SUPPORT = NO
-
-# For Microsoft's IDL there are propget and propput attributes to indicate getter
-# and setter methods for a property. Setting this option to YES (the default)
-# will make doxygen to replace the get and set methods by a property in the
-# documentation. This will only work if the methods are indeed getting or
-# setting a simple type. If this is not the case, or you want to show the
-# methods anyway, you should set this option to NO.
-
-IDL_PROPERTY_SUPPORT = YES
-
-# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC
-# tag is set to YES, then doxygen will reuse the documentation of the first
-# member in the group (if any) for the other members of the group. By default
-# all members of a group must be documented explicitly.
-
-DISTRIBUTE_GROUP_DOC = NO
-
-# Set the SUBGROUPING tag to YES (the default) to allow class member groups of
-# the same type (for instance a group of public functions) to be put as a
-# subgroup of that type (e.g. under the Public Functions section). Set it to
-# NO to prevent subgrouping. Alternatively, this can be done per class using
-# the \nosubgrouping command.
-
-SUBGROUPING = YES
-
-# When TYPEDEF_HIDES_STRUCT is enabled, a typedef of a struct, union, or enum
-# is documented as struct, union, or enum with the name of the typedef. So
-# typedef struct TypeS {} TypeT, will appear in the documentation as a struct
-# with name TypeT. When disabled the typedef will appear as a member of a file,
-# namespace, or class. And the struct will be named TypeS. This can typically
-# be useful for C code in case the coding convention dictates that all compound
-# types are typedef'ed and only the typedef is referenced, never the tag name.
-
-TYPEDEF_HIDES_STRUCT = NO
-
-# The SYMBOL_CACHE_SIZE determines the size of the internal cache use to
-# determine which symbols to keep in memory and which to flush to disk.
-# When the cache is full, less often used symbols will be written to disk.
-# For small to medium size projects (<1000 input files) the default value is
-# probably good enough. For larger projects a too small cache size can cause
-# doxygen to be busy swapping symbols to and from disk most of the time
-# causing a significant performance penality.
-# If the system has enough physical memory increasing the cache will improve the
-# performance by keeping more symbols in memory. Note that the value works on
-# a logarithmic scale so increasing the size by one will rougly double the
-# memory usage. The cache size is given by this formula:
-# 2^(16+SYMBOL_CACHE_SIZE). The valid range is 0..9, the default is 0,
-# corresponding to a cache size of 2^16 = 65536 symbols
-
-SYMBOL_CACHE_SIZE = 0
-
-#---------------------------------------------------------------------------
-# Build related configuration options
-#---------------------------------------------------------------------------
-
-# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in
-# documentation are documented, even if no documentation was available.
-# Private class members and static file members will be hidden unless
-# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES
-
-EXTRACT_ALL = NO
-
-# If the EXTRACT_PRIVATE tag is set to YES all private members of a class
-# will be included in the documentation.
-
-EXTRACT_PRIVATE = YES
-
-# If the EXTRACT_STATIC tag is set to YES all static members of a file
-# will be included in the documentation.
-
-EXTRACT_STATIC = NO
-
-# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs)
-# defined locally in source files will be included in the documentation.
-# If set to NO only classes defined in header files are included.
-
-EXTRACT_LOCAL_CLASSES = YES
-
-# This flag is only useful for Objective-C code. When set to YES local
-# methods, which are defined in the implementation section but not in
-# the interface are included in the documentation.
-# If set to NO (the default) only methods in the interface are included.
-
-EXTRACT_LOCAL_METHODS = NO
-
-# If this flag is set to YES, the members of anonymous namespaces will be
-# extracted and appear in the documentation as a namespace called
-# 'anonymous_namespace{file}', where file will be replaced with the base
-# name of the file that contains the anonymous namespace. By default
-# anonymous namespace are hidden.
-
-EXTRACT_ANON_NSPACES = NO
-
-# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all
-# undocumented members of documented classes, files or namespaces.
-# If set to NO (the default) these members will be included in the
-# various overviews, but no documentation section is generated.
-# This option has no effect if EXTRACT_ALL is enabled.
-
-HIDE_UNDOC_MEMBERS = YES
-
-# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all
-# undocumented classes that are normally visible in the class hierarchy.
-# If set to NO (the default) these classes will be included in the various
-# overviews. This option has no effect if EXTRACT_ALL is enabled.
-
-HIDE_UNDOC_CLASSES = YES
-
-# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all
-# friend (class|struct|union) declarations.
-# If set to NO (the default) these declarations will be included in the
-# documentation.
-
-HIDE_FRIEND_COMPOUNDS = NO
-
-# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any
-# documentation blocks found inside the body of a function.
-# If set to NO (the default) these blocks will be appended to the
-# function's detailed documentation block.
-
-HIDE_IN_BODY_DOCS = NO
-
-# The INTERNAL_DOCS tag determines if documentation
-# that is typed after a \internal command is included. If the tag is set
-# to NO (the default) then the documentation will be excluded.
-# Set it to YES to include the internal documentation.
-
-INTERNAL_DOCS = NO
-
-# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate
-# file names in lower-case letters. If set to YES upper-case letters are also
-# allowed. This is useful if you have classes or files whose names only differ
-# in case and if your file system supports case sensitive file names. Windows
-# and Mac users are advised to set this option to NO.
-
-CASE_SENSE_NAMES = YES
-
-# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen
-# will show members with their full class and namespace scopes in the
-# documentation. If set to YES the scope will be hidden.
-
-HIDE_SCOPE_NAMES = NO
-
-# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen
-# will put a list of the files that are included by a file in the documentation
-# of that file.
-
-SHOW_INCLUDE_FILES = YES
-
-# If the INLINE_INFO tag is set to YES (the default) then a tag [inline]
-# is inserted in the documentation for inline members.
-
-INLINE_INFO = YES
-
-# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen
-# will sort the (detailed) documentation of file and class members
-# alphabetically by member name. If set to NO the members will appear in
-# declaration order.
-
-SORT_MEMBER_DOCS = YES
-
-# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the
-# brief documentation of file, namespace and class members alphabetically
-# by member name. If set to NO (the default) the members will appear in
-# declaration order.
-
-SORT_BRIEF_DOCS = NO
-
-# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the (brief and detailed) documentation of class members so that constructors and destructors are listed first. If set to NO (the default) the constructors will appear in the respective orders defined by SORT_MEMBER_DOCS and SORT_BRIEF_DOCS. This tag will be ignored for brief docs if SORT_BRIEF_DOCS is set to NO and ignored for detailed docs if SORT_MEMBER_DOCS is set to NO.
-
-SORT_MEMBERS_CTORS_1ST = YES
-
-# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the
-# hierarchy of group names into alphabetical order. If set to NO (the default)
-# the group names will appear in their defined order.
-
-SORT_GROUP_NAMES = NO
-
-# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be
-# sorted by fully-qualified names, including namespaces. If set to
-# NO (the default), the class list will be sorted only by class name,
-# not including the namespace part.
-# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES.
-# Note: This option applies only to the class list, not to the
-# alphabetical list.
-
-SORT_BY_SCOPE_NAME = YES
-
-# The GENERATE_TODOLIST tag can be used to enable (YES) or
-# disable (NO) the todo list. This list is created by putting \todo
-# commands in the documentation.
-
-GENERATE_TODOLIST = YES
-
-# The GENERATE_TESTLIST tag can be used to enable (YES) or
-# disable (NO) the test list. This list is created by putting \test
-# commands in the documentation.
-
-GENERATE_TESTLIST = YES
-
-# The GENERATE_BUGLIST tag can be used to enable (YES) or
-# disable (NO) the bug list. This list is created by putting \bug
-# commands in the documentation.
-
-GENERATE_BUGLIST = YES
-
-# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or
-# disable (NO) the deprecated list. This list is created by putting
-# \deprecated commands in the documentation.
-
-GENERATE_DEPRECATEDLIST= YES
-
-# The ENABLED_SECTIONS tag can be used to enable conditional
-# documentation sections, marked by \if sectionname ... \endif.
-
-ENABLED_SECTIONS =
-
-# The MAX_INITIALIZER_LINES tag determines the maximum number of lines
-# the initial value of a variable or define consists of for it to appear in
-# the documentation. If the initializer consists of more lines than specified
-# here it will be hidden. Use a value of 0 to hide initializers completely.
-# The appearance of the initializer of individual variables and defines in the
-# documentation can be controlled using \showinitializer or \hideinitializer
-# command in the documentation regardless of this setting.
-
-MAX_INITIALIZER_LINES = 30
-
-# Set the SHOW_USED_FILES tag to NO to disable the list of files generated
-# at the bottom of the documentation of classes and structs. If set to YES the
-# list will mention the files that were used to generate the documentation.
-
-SHOW_USED_FILES = YES
-
-# If the sources in your project are distributed over multiple directories
-# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy
-# in the documentation. The default is NO.
-
-SHOW_DIRECTORIES = NO
-
-# Set the SHOW_FILES tag to NO to disable the generation of the Files page.
-# This will remove the Files entry from the Quick Index and from the
-# Folder Tree View (if specified). The default is YES.
-
-SHOW_FILES = YES
-
-# Set the SHOW_NAMESPACES tag to NO to disable the generation of the
-# Namespaces page.
-# This will remove the Namespaces entry from the Quick Index
-# and from the Folder Tree View (if specified). The default is YES.
-
-SHOW_NAMESPACES = YES
-
-# The FILE_VERSION_FILTER tag can be used to specify a program or script that
-# doxygen should invoke to get the current version for each file (typically from
-# the version control system). Doxygen will invoke the program by executing (via
-# popen()) the command <command> <input-file>, where <command> is the value of
-# the FILE_VERSION_FILTER tag, and <input-file> is the name of an input file
-# provided by doxygen. Whatever the program writes to standard output
-# is used as the file version. See the manual for examples.
-
-FILE_VERSION_FILTER =
-
-# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed by
-# doxygen. The layout file controls the global structure of the generated output files
-# in an output format independent way. The create the layout file that represents
-# doxygen's defaults, run doxygen with the -l option. You can optionally specify a
-# file name after the option, if omitted DoxygenLayout.xml will be used as the name
-# of the layout file.
-
-LAYOUT_FILE =
-
-#---------------------------------------------------------------------------
-# configuration options related to warning and progress messages
-#---------------------------------------------------------------------------
-
-# The QUIET tag can be used to turn on/off the messages that are generated
-# by doxygen. Possible values are YES and NO. If left blank NO is used.
-
-QUIET = NO
-
-# The WARNINGS tag can be used to turn on/off the warning messages that are
-# generated by doxygen. Possible values are YES and NO. If left blank
-# NO is used.
-
-WARNINGS = YES
-
-# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings
-# for undocumented members. If EXTRACT_ALL is set to YES then this flag will
-# automatically be disabled.
-
-WARN_IF_UNDOCUMENTED = YES
-
-# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for
-# potential errors in the documentation, such as not documenting some
-# parameters in a documented function, or documenting parameters that
-# don't exist or using markup commands wrongly.
-
-WARN_IF_DOC_ERROR = YES
-
-# This WARN_NO_PARAMDOC option can be abled to get warnings for
-# functions that are documented, but have no documentation for their parameters
-# or return value. If set to NO (the default) doxygen will only warn about
-# wrong or incomplete parameter documentation, but not about the absence of
-# documentation.
-
-WARN_NO_PARAMDOC = NO
-
-# The WARN_FORMAT tag determines the format of the warning messages that
-# doxygen can produce. The string should contain the $file, $line, and $text
-# tags, which will be replaced by the file and line number from which the
-# warning originated and the warning text. Optionally the format may contain
-# $version, which will be replaced by the version of the file (if it could
-# be obtained via FILE_VERSION_FILTER)
-
-WARN_FORMAT = "$file:$line: $text"
-
-# The WARN_LOGFILE tag can be used to specify a file to which warning
-# and error messages should be written. If left blank the output is written
-# to stderr.
-
-WARN_LOGFILE =
-
-#---------------------------------------------------------------------------
-# configuration options related to the input files
-#---------------------------------------------------------------------------
-
-# The INPUT tag can be used to specify the files and/or directories that contain
-# documented source files. You may enter file names like "myfile.cpp" or
-# directories like "/usr/src/myproject". Separate the files or directories
-# with spaces.
-
-INPUT = lm lm/builder lm/filter lm/interpolate lm/wrappers util util/double-conversion util/stream
-
-# This tag can be used to specify the character encoding of the source files
-# that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is
-# also the default input encoding. Doxygen uses libiconv (or the iconv built
-# into libc) for the transcoding. See http://www.gnu.org/software/libiconv for
-# the list of possible encodings.
-
-INPUT_ENCODING = UTF-8
-
-# If the value of the INPUT tag contains directories, you can use the
-# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp
-# and *.h) to filter out the source-files in the directories. If left
-# blank the following patterns are tested:
-# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx
-# *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py *.f90
-
-FILE_PATTERNS =
-
-# The RECURSIVE tag can be used to turn specify whether or not subdirectories
-# should be searched for input files as well. Possible values are YES and NO.
-# If left blank NO is used.
-
-RECURSIVE = NO
-
-# The EXCLUDE tag can be used to specify files and/or directories that should
-# excluded from the INPUT source files. This way you can easily exclude a
-# subdirectory from a directory tree whose root is specified with the INPUT tag.
-
-EXCLUDE =
-
-# The EXCLUDE_SYMLINKS tag can be used select whether or not files or
-# directories that are symbolic links (a Unix filesystem feature) are excluded
-# from the input.
-
-EXCLUDE_SYMLINKS = NO
-
-# If the value of the INPUT tag contains directories, you can use the
-# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude
-# certain files from those directories. Note that the wildcards are matched
-# against the file with absolute path, so to exclude all test directories
-# for example use the pattern */test/*
-
-EXCLUDE_PATTERNS =
-
-# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names
-# (namespaces, classes, functions, etc.) that should be excluded from the
-# output. The symbol name can be a fully qualified name, a word, or if the
-# wildcard * is used, a substring. Examples: ANamespace, AClass,
-# AClass::ANamespace, ANamespace::*Test
-
-EXCLUDE_SYMBOLS =
-
-# The EXAMPLE_PATH tag can be used to specify one or more files or
-# directories that contain example code fragments that are included (see
-# the \include command).
-
-EXAMPLE_PATH =
-
-# If the value of the EXAMPLE_PATH tag contains directories, you can use the
-# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp
-# and *.h) to filter out the source-files in the directories. If left
-# blank all files are included.
-
-EXAMPLE_PATTERNS =
-
-# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be
-# searched for input files to be used with the \include or \dontinclude
-# commands irrespective of the value of the RECURSIVE tag.
-# Possible values are YES and NO. If left blank NO is used.
-
-EXAMPLE_RECURSIVE = NO
-
-# The IMAGE_PATH tag can be used to specify one or more files or
-# directories that contain image that are included in the documentation (see
-# the \image command).
-
-IMAGE_PATH =
-
-# The INPUT_FILTER tag can be used to specify a program that doxygen should
-# invoke to filter for each input file. Doxygen will invoke the filter program
-# by executing (via popen()) the command <filter> <input-file>, where <filter>
-# is the value of the INPUT_FILTER tag, and <input-file> is the name of an
-# input file. Doxygen will then use the output that the filter program writes
-# to standard output.
-# If FILTER_PATTERNS is specified, this tag will be
-# ignored.
-
-INPUT_FILTER =
-
-# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern
-# basis.
-# Doxygen will compare the file name with each pattern and apply the
-# filter if there is a match.
-# The filters are a list of the form:
-# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further
-# info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER
-# is applied to all files.
-
-FILTER_PATTERNS =
-
-# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using
-# INPUT_FILTER) will be used to filter the input files when producing source
-# files to browse (i.e. when SOURCE_BROWSER is set to YES).
-
-FILTER_SOURCE_FILES = NO
-
-#---------------------------------------------------------------------------
-# configuration options related to source browsing
-#---------------------------------------------------------------------------
-
-# If the SOURCE_BROWSER tag is set to YES then a list of source files will
-# be generated. Documented entities will be cross-referenced with these sources.
-# Note: To get rid of all source code in the generated output, make sure also
-# VERBATIM_HEADERS is set to NO.
-
-SOURCE_BROWSER = NO
-
-# Setting the INLINE_SOURCES tag to YES will include the body
-# of functions and classes directly in the documentation.
-
-INLINE_SOURCES = NO
-
-# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct
-# doxygen to hide any special comment blocks from generated source code
-# fragments. Normal C and C++ comments will always remain visible.
-
-STRIP_CODE_COMMENTS = YES
-
-# If the REFERENCED_BY_RELATION tag is set to YES
-# then for each documented function all documented
-# functions referencing it will be listed.
-
-REFERENCED_BY_RELATION = NO
-
-# If the REFERENCES_RELATION tag is set to YES
-# then for each documented function all documented entities
-# called/used by that function will be listed.
-
-REFERENCES_RELATION = NO
-
-# If the REFERENCES_LINK_SOURCE tag is set to YES (the default)
-# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from
-# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will
-# link to the source code.
-# Otherwise they will link to the documentation.
-
-REFERENCES_LINK_SOURCE = YES
-
-# If the USE_HTAGS tag is set to YES then the references to source code
-# will point to the HTML generated by the htags(1) tool instead of doxygen
-# built-in source browser. The htags tool is part of GNU's global source
-# tagging system (see http://www.gnu.org/software/global/global.html). You
-# will need version 4.8.6 or higher.
-
-USE_HTAGS = NO
-
-# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen
-# will generate a verbatim copy of the header file for each class for
-# which an include is specified. Set to NO to disable this.
-
-VERBATIM_HEADERS = YES
-
-#---------------------------------------------------------------------------
-# configuration options related to the alphabetical class index
-#---------------------------------------------------------------------------
-
-# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index
-# of all compounds will be generated. Enable this if the project
-# contains a lot of classes, structs, unions or interfaces.
-
-ALPHABETICAL_INDEX = NO
-
-# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then
-# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns
-# in which this list will be split (can be a number in the range [1..20])
-
-COLS_IN_ALPHA_INDEX = 5
-
-# In case all classes in a project start with a common prefix, all
-# classes will be put under the same header in the alphabetical index.
-# The IGNORE_PREFIX tag can be used to specify one or more prefixes that
-# should be ignored while generating the index headers.
-
-IGNORE_PREFIX =
-
-#---------------------------------------------------------------------------
-# configuration options related to the HTML output
-#---------------------------------------------------------------------------
-
-# If the GENERATE_HTML tag is set to YES (the default) Doxygen will
-# generate HTML output.
-
-GENERATE_HTML = YES
-
-# The HTML_OUTPUT tag is used to specify where the HTML docs will be put.
-# If a relative path is entered the value of OUTPUT_DIRECTORY will be
-# put in front of it. If left blank `html' will be used as the default path.
-
-HTML_OUTPUT = html
-
-# The HTML_FILE_EXTENSION tag can be used to specify the file extension for
-# each generated HTML page (for example: .htm,.php,.asp). If it is left blank
-# doxygen will generate files with .html extension.
-
-HTML_FILE_EXTENSION = .html
-
-# The HTML_HEADER tag can be used to specify a personal HTML header for
-# each generated HTML page. If it is left blank doxygen will generate a
-# standard header.
-
-HTML_HEADER =
-
-# The HTML_FOOTER tag can be used to specify a personal HTML footer for
-# each generated HTML page. If it is left blank doxygen will generate a
-# standard footer.
-
-HTML_FOOTER =
-
-# If the HTML_TIMESTAMP tag is set to YES then the generated HTML
-# documentation will contain the timesstamp.
-
-HTML_TIMESTAMP = NO
-
-# The HTML_STYLESHEET tag can be used to specify a user-defined cascading
-# style sheet that is used by each HTML page. It can be used to
-# fine-tune the look of the HTML output. If the tag is left blank doxygen
-# will generate a default style sheet. Note that doxygen will try to copy
-# the style sheet file to the HTML output directory, so don't put your own
-# stylesheet in the HTML output directory as well, or it will be erased!
-
-HTML_STYLESHEET =
-
-# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes,
-# files or namespaces will be aligned in HTML using tables. If set to
-# NO a bullet list will be used.
-
-HTML_ALIGN_MEMBERS = YES
-
-# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML
-# documentation will contain sections that can be hidden and shown after the
-# page has loaded. For this to work a browser that supports
-# JavaScript and DHTML is required (for instance Mozilla 1.0+, Firefox
-# Netscape 6.0+, Internet explorer 5.0+, Konqueror, or Safari).
-
-HTML_DYNAMIC_SECTIONS = NO
-
-# If the GENERATE_DOCSET tag is set to YES, additional index files
-# will be generated that can be used as input for Apple's Xcode 3
-# integrated development environment, introduced with OSX 10.5 (Leopard).
-# To create a documentation set, doxygen will generate a Makefile in the
-# HTML output directory. Running make will produce the docset in that
-# directory and running "make install" will install the docset in
-# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find
-# it at startup.
-# See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html for more information.
-
-GENERATE_DOCSET = NO
-
-# When GENERATE_DOCSET tag is set to YES, this tag determines the name of the
-# feed. A documentation feed provides an umbrella under which multiple
-# documentation sets from a single provider (such as a company or product suite)
-# can be grouped.
-
-DOCSET_FEEDNAME = "Doxygen generated docs"
-
-# When GENERATE_DOCSET tag is set to YES, this tag specifies a string that
-# should uniquely identify the documentation set bundle. This should be a
-# reverse domain-name style string, e.g. com.mycompany.MyDocSet. Doxygen
-# will append .docset to the name.
-
-DOCSET_BUNDLE_ID = org.doxygen.Project
-
-# If the GENERATE_HTMLHELP tag is set to YES, additional index files
-# will be generated that can be used as input for tools like the
-# Microsoft HTML help workshop to generate a compiled HTML help file (.chm)
-# of the generated HTML documentation.
-
-GENERATE_HTMLHELP = NO
-
-# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can
-# be used to specify the file name of the resulting .chm file. You
-# can add a path in front of the file if the result should not be
-# written to the html output directory.
-
-CHM_FILE =
-
-# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can
-# be used to specify the location (absolute path including file name) of
-# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run
-# the HTML help compiler on the generated index.hhp.
-
-HHC_LOCATION =
-
-# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag
-# controls if a separate .chi index file is generated (YES) or that
-# it should be included in the master .chm file (NO).
-
-GENERATE_CHI = NO
-
-# If the GENERATE_HTMLHELP tag is set to YES, the CHM_INDEX_ENCODING
-# is used to encode HtmlHelp index (hhk), content (hhc) and project file
-# content.
-
-CHM_INDEX_ENCODING =
-
-# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag
-# controls whether a binary table of contents is generated (YES) or a
-# normal table of contents (NO) in the .chm file.
-
-BINARY_TOC = NO
-
-# The TOC_EXPAND flag can be set to YES to add extra items for group members
-# to the contents of the HTML help documentation and to the tree view.
-
-TOC_EXPAND = NO
-
-# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and QHP_VIRTUAL_FOLDER
-# are set, an additional index file will be generated that can be used as input for
-# Qt's qhelpgenerator to generate a Qt Compressed Help (.qch) of the generated
-# HTML documentation.
-
-GENERATE_QHP = NO
-
-# If the QHG_LOCATION tag is specified, the QCH_FILE tag can
-# be used to specify the file name of the resulting .qch file.
-# The path specified is relative to the HTML output folder.
-
-QCH_FILE =
-
-# The QHP_NAMESPACE tag specifies the namespace to use when generating
-# Qt Help Project output. For more information please see
-# http://doc.trolltech.com/qthelpproject.html#namespace
-
-QHP_NAMESPACE =
-
-# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating
-# Qt Help Project output. For more information please see
-# http://doc.trolltech.com/qthelpproject.html#virtual-folders
-
-QHP_VIRTUAL_FOLDER = doc
-
-# If QHP_CUST_FILTER_NAME is set, it specifies the name of a custom filter to add.
-# For more information please see
-# http://doc.trolltech.com/qthelpproject.html#custom-filters
-
-QHP_CUST_FILTER_NAME =
-
-# The QHP_CUST_FILT_ATTRS tag specifies the list of the attributes of the custom filter to add.For more information please see
-# <a href="http://doc.trolltech.com/qthelpproject.html#custom-filters">Qt Help Project / Custom Filters</a>.
-
-QHP_CUST_FILTER_ATTRS =
-
-# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this project's
-# filter section matches.
-# <a href="http://doc.trolltech.com/qthelpproject.html#filter-attributes">Qt Help Project / Filter Attributes</a>.
-
-QHP_SECT_FILTER_ATTRS =
-
-# If the GENERATE_QHP tag is set to YES, the QHG_LOCATION tag can
-# be used to specify the location of Qt's qhelpgenerator.
-# If non-empty doxygen will try to run qhelpgenerator on the generated
-# .qhp file.
-
-QHG_LOCATION =
-
-# The DISABLE_INDEX tag can be used to turn on/off the condensed index at
-# top of each HTML page. The value NO (the default) enables the index and
-# the value YES disables it.
-
-DISABLE_INDEX = NO
-
-# This tag can be used to set the number of enum values (range [1..20])
-# that doxygen will group on one line in the generated HTML documentation.
-
-ENUM_VALUES_PER_LINE = 4
-
-# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index
-# structure should be generated to display hierarchical information.
-# If the tag value is set to YES, a side panel will be generated
-# containing a tree-like index structure (just like the one that
-# is generated for HTML Help). For this to work a browser that supports
-# JavaScript, DHTML, CSS and frames is required (i.e. any modern browser).
-# Windows users are probably better off using the HTML help feature.
-
-GENERATE_TREEVIEW = NO
-
-# By enabling USE_INLINE_TREES, doxygen will generate the Groups, Directories,
-# and Class Hierarchy pages using a tree view instead of an ordered list.
-
-USE_INLINE_TREES = NO
-
-# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be
-# used to set the initial width (in pixels) of the frame in which the tree
-# is shown.
-
-TREEVIEW_WIDTH = 250
-
-# Use this tag to change the font size of Latex formulas included
-# as images in the HTML documentation. The default is 10. Note that
-# when you change the font size after a successful doxygen run you need
-# to manually remove any form_*.png images from the HTML output directory
-# to force them to be regenerated.
-
-FORMULA_FONTSIZE = 10
-
-# When the SEARCHENGINE tag is enable doxygen will generate a search box for the HTML output. The underlying search engine uses javascript
-# and DHTML and should work on any modern browser. Note that when using HTML help (GENERATE_HTMLHELP) or Qt help (GENERATE_QHP)
-# there is already a search function so this one should typically
-# be disabled.
-
-SEARCHENGINE = YES
-
-#---------------------------------------------------------------------------
-# configuration options related to the LaTeX output
-#---------------------------------------------------------------------------
-
-# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will
-# generate Latex output.
-
-GENERATE_LATEX = YES
-
-# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put.
-# If a relative path is entered the value of OUTPUT_DIRECTORY will be
-# put in front of it. If left blank `latex' will be used as the default path.
-
-LATEX_OUTPUT = latex
-
-# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be
-# invoked. If left blank `latex' will be used as the default command name.
-
-LATEX_CMD_NAME = latex
-
-# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to
-# generate index for LaTeX. If left blank `makeindex' will be used as the
-# default command name.
-
-MAKEINDEX_CMD_NAME = makeindex
-
-# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact
-# LaTeX documents. This may be useful for small projects and may help to
-# save some trees in general.
-
-COMPACT_LATEX = NO
-
-# The PAPER_TYPE tag can be used to set the paper type that is used
-# by the printer. Possible values are: a4, a4wide, letter, legal and
-# executive. If left blank a4wide will be used.
-
-PAPER_TYPE = a4wide
-
-# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX
-# packages that should be included in the LaTeX output.
-
-EXTRA_PACKAGES =
-
-# The LATEX_HEADER tag can be used to specify a personal LaTeX header for
-# the generated latex document. The header should contain everything until
-# the first chapter. If it is left blank doxygen will generate a
-# standard header. Notice: only use this tag if you know what you are doing!
-
-LATEX_HEADER =
-
-# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated
-# is prepared for conversion to pdf (using ps2pdf). The pdf file will
-# contain links (just like the HTML output) instead of page references
-# This makes the output suitable for online browsing using a pdf viewer.
-
-PDF_HYPERLINKS = YES
-
-# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of
-# plain latex in the generated Makefile. Set this option to YES to get a
-# higher quality PDF documentation.
-
-USE_PDFLATEX = YES
-
-# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode.
-# command to the generated LaTeX files. This will instruct LaTeX to keep
-# running if errors occur, instead of asking the user for help.
-# This option is also used when generating formulas in HTML.
-
-LATEX_BATCHMODE = NO
-
-# If LATEX_HIDE_INDICES is set to YES then doxygen will not
-# include the index chapters (such as File Index, Compound Index, etc.)
-# in the output.
-
-LATEX_HIDE_INDICES = NO
-
-# If LATEX_SOURCE_CODE is set to YES then doxygen will include source code with syntax highlighting in the LaTeX output. Note that which sources are shown also depends on other settings such as SOURCE_BROWSER.
-
-LATEX_SOURCE_CODE = NO
-
-#---------------------------------------------------------------------------
-# configuration options related to the RTF output
-#---------------------------------------------------------------------------
-
-# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output
-# The RTF output is optimized for Word 97 and may not look very pretty with
-# other RTF readers or editors.
-
-GENERATE_RTF = NO
-
-# The RTF_OUTPUT tag is used to specify where the RTF docs will be put.
-# If a relative path is entered the value of OUTPUT_DIRECTORY will be
-# put in front of it. If left blank `rtf' will be used as the default path.
-
-RTF_OUTPUT = rtf
-
-# If the COMPACT_RTF tag is set to YES Doxygen generates more compact
-# RTF documents. This may be useful for small projects and may help to
-# save some trees in general.
-
-COMPACT_RTF = NO
-
-# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated
-# will contain hyperlink fields. The RTF file will
-# contain links (just like the HTML output) instead of page references.
-# This makes the output suitable for online browsing using WORD or other
-# programs which support those fields.
-# Note: wordpad (write) and others do not support links.
-
-RTF_HYPERLINKS = NO
-
-# Load stylesheet definitions from file. Syntax is similar to doxygen's
-# config file, i.e. a series of assignments. You only have to provide
-# replacements, missing definitions are set to their default value.
-
-RTF_STYLESHEET_FILE =
-
-# Set optional variables used in the generation of an rtf document.
-# Syntax is similar to doxygen's config file.
-
-RTF_EXTENSIONS_FILE =
-
-#---------------------------------------------------------------------------
-# configuration options related to the man page output
-#---------------------------------------------------------------------------
-
-# If the GENERATE_MAN tag is set to YES (the default) Doxygen will
-# generate man pages
-
-GENERATE_MAN = NO
-
-# The MAN_OUTPUT tag is used to specify where the man pages will be put.
-# If a relative path is entered the value of OUTPUT_DIRECTORY will be
-# put in front of it. If left blank `man' will be used as the default path.
-
-MAN_OUTPUT = man
-
-# The MAN_EXTENSION tag determines the extension that is added to
-# the generated man pages (default is the subroutine's section .3)
-
-MAN_EXTENSION = .3
-
-# If the MAN_LINKS tag is set to YES and Doxygen generates man output,
-# then it will generate one additional man file for each entity
-# documented in the real man page(s). These additional files
-# only source the real man page, but without them the man command
-# would be unable to find the correct page. The default is NO.
-
-MAN_LINKS = NO
-
-#---------------------------------------------------------------------------
-# configuration options related to the XML output
-#---------------------------------------------------------------------------
-
-# If the GENERATE_XML tag is set to YES Doxygen will
-# generate an XML file that captures the structure of
-# the code including all documentation.
-
-GENERATE_XML = NO
-
-# The XML_OUTPUT tag is used to specify where the XML pages will be put.
-# If a relative path is entered the value of OUTPUT_DIRECTORY will be
-# put in front of it. If left blank `xml' will be used as the default path.
-
-XML_OUTPUT = xml
-
-# The XML_SCHEMA tag can be used to specify an XML schema,
-# which can be used by a validating XML parser to check the
-# syntax of the XML files.
-
-XML_SCHEMA =
-
-# The XML_DTD tag can be used to specify an XML DTD,
-# which can be used by a validating XML parser to check the
-# syntax of the XML files.
-
-XML_DTD =
-
-# If the XML_PROGRAMLISTING tag is set to YES Doxygen will
-# dump the program listings (including syntax highlighting
-# and cross-referencing information) to the XML output. Note that
-# enabling this will significantly increase the size of the XML output.
-
-XML_PROGRAMLISTING = YES
-
-#---------------------------------------------------------------------------
-# configuration options for the AutoGen Definitions output
-#---------------------------------------------------------------------------
-
-# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will
-# generate an AutoGen Definitions (see autogen.sf.net) file
-# that captures the structure of the code including all
-# documentation. Note that this feature is still experimental
-# and incomplete at the moment.
-
-GENERATE_AUTOGEN_DEF = NO
-
-#---------------------------------------------------------------------------
-# configuration options related to the Perl module output
-#---------------------------------------------------------------------------
-
-# If the GENERATE_PERLMOD tag is set to YES Doxygen will
-# generate a Perl module file that captures the structure of
-# the code including all documentation. Note that this
-# feature is still experimental and incomplete at the
-# moment.
-
-GENERATE_PERLMOD = NO
-
-# If the PERLMOD_LATEX tag is set to YES Doxygen will generate
-# the necessary Makefile rules, Perl scripts and LaTeX code to be able
-# to generate PDF and DVI output from the Perl module output.
-
-PERLMOD_LATEX = NO
-
-# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be
-# nicely formatted so it can be parsed by a human reader.
-# This is useful
-# if you want to understand what is going on.
-# On the other hand, if this
-# tag is set to NO the size of the Perl module output will be much smaller
-# and Perl will parse it just the same.
-
-PERLMOD_PRETTY = YES
-
-# The names of the make variables in the generated doxyrules.make file
-# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX.
-# This is useful so different doxyrules.make files included by the same
-# Makefile don't overwrite each other's variables.
-
-PERLMOD_MAKEVAR_PREFIX =
-
-#---------------------------------------------------------------------------
-# Configuration options related to the preprocessor
-#---------------------------------------------------------------------------
-
-# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will
-# evaluate all C-preprocessor directives found in the sources and include
-# files.
-
-ENABLE_PREPROCESSING = YES
-
-# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro
-# names in the source code. If set to NO (the default) only conditional
-# compilation will be performed. Macro expansion can be done in a controlled
-# way by setting EXPAND_ONLY_PREDEF to YES.
-
-MACRO_EXPANSION = NO
-
-# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES
-# then the macro expansion is limited to the macros specified with the
-# PREDEFINED and EXPAND_AS_DEFINED tags.
-
-EXPAND_ONLY_PREDEF = NO
-
-# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files
-# in the INCLUDE_PATH (see below) will be search if a #include is found.
-
-SEARCH_INCLUDES = YES
-
-# The INCLUDE_PATH tag can be used to specify one or more directories that
-# contain include files that are not input files but should be processed by
-# the preprocessor.
-
-INCLUDE_PATH =
-
-# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard
-# patterns (like *.h and *.hpp) to filter out the header-files in the
-# directories. If left blank, the patterns specified with FILE_PATTERNS will
-# be used.
-
-INCLUDE_FILE_PATTERNS =
-
-# The PREDEFINED tag can be used to specify one or more macro names that
-# are defined before the preprocessor is started (similar to the -D option of
-# gcc). The argument of the tag is a list of macros of the form: name
-# or name=definition (no spaces). If the definition and the = are
-# omitted =1 is assumed. To prevent a macro definition from being
-# undefined via #undef or recursively expanded use the := operator
-# instead of the = operator.
-
-PREDEFINED =
-
-# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then
-# this tag can be used to specify a list of macro names that should be expanded.
-# The macro definition that is found in the sources will be used.
-# Use the PREDEFINED tag if you want to use a different macro definition.
-
-EXPAND_AS_DEFINED =
-
-# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then
-# doxygen's preprocessor will remove all function-like macros that are alone
-# on a line, have an all uppercase name, and do not end with a semicolon. Such
-# function macros are typically used for boiler-plate code, and will confuse
-# the parser if not removed.
-
-SKIP_FUNCTION_MACROS = YES
-
-#---------------------------------------------------------------------------
-# Configuration::additions related to external references
-#---------------------------------------------------------------------------
-
-# The TAGFILES option can be used to specify one or more tagfiles.
-# Optionally an initial location of the external documentation
-# can be added for each tagfile. The format of a tag file without
-# this location is as follows:
-#
-# TAGFILES = file1 file2 ...
-# Adding location for the tag files is done as follows:
-#
-# TAGFILES = file1=loc1 "file2 = loc2" ...
-# where "loc1" and "loc2" can be relative or absolute paths or
-# URLs. If a location is present for each tag, the installdox tool
-# does not have to be run to correct the links.
-# Note that each tag file must have a unique name
-# (where the name does NOT include the path)
-# If a tag file is not located in the directory in which doxygen
-# is run, you must also specify the path to the tagfile here.
-
-TAGFILES =
-
-# When a file name is specified after GENERATE_TAGFILE, doxygen will create
-# a tag file that is based on the input files it reads.
-
-GENERATE_TAGFILE =
-
-# If the ALLEXTERNALS tag is set to YES all external classes will be listed
-# in the class index. If set to NO only the inherited external classes
-# will be listed.
-
-ALLEXTERNALS = NO
-
-# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed
-# in the modules index. If set to NO, only the current project's groups will
-# be listed.
-
-EXTERNAL_GROUPS = YES
-
-# The PERL_PATH should be the absolute path and name of the perl script
-# interpreter (i.e. the result of `which perl').
-
-PERL_PATH = /usr/bin/perl
-
-#---------------------------------------------------------------------------
-# Configuration options related to the dot tool
-#---------------------------------------------------------------------------
-
-# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will
-# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base
-# or super classes. Setting the tag to NO turns the diagrams off. Note that
-# this option is superseded by the HAVE_DOT option below. This is only a
-# fallback. It is recommended to install and use dot, since it yields more
-# powerful graphs.
-
-CLASS_DIAGRAMS = YES
-
-# You can define message sequence charts within doxygen comments using the \msc
-# command. Doxygen will then run the mscgen tool (see
-# http://www.mcternan.me.uk/mscgen/) to produce the chart and insert it in the
-# documentation. The MSCGEN_PATH tag allows you to specify the directory where
-# the mscgen tool resides. If left empty the tool is assumed to be found in the
-# default search path.
-
-MSCGEN_PATH =
-
-# If set to YES, the inheritance and collaboration graphs will hide
-# inheritance and usage relations if the target is undocumented
-# or is not a class.
-
-HIDE_UNDOC_RELATIONS = YES
-
-# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is
-# available from the path. This tool is part of Graphviz, a graph visualization
-# toolkit from AT&T and Lucent Bell Labs. The other options in this section
-# have no effect if this option is set to NO (the default)
-
-HAVE_DOT = NO
-
-# By default doxygen will write a font called FreeSans.ttf to the output
-# directory and reference it in all dot files that doxygen generates. This
-# font does not include all possible unicode characters however, so when you need
-# these (or just want a differently looking font) you can specify the font name
-# using DOT_FONTNAME. You need need to make sure dot is able to find the font,
-# which can be done by putting it in a standard location or by setting the
-# DOTFONTPATH environment variable or by setting DOT_FONTPATH to the directory
-# containing the font.
-
-DOT_FONTNAME = FreeSans
-
-# The DOT_FONTSIZE tag can be used to set the size of the font of dot graphs.
-# The default size is 10pt.
-
-DOT_FONTSIZE = 10
-
-# By default doxygen will tell dot to use the output directory to look for the
-# FreeSans.ttf font (which doxygen will put there itself). If you specify a
-# different font using DOT_FONTNAME you can set the path where dot
-# can find it using this tag.
-
-DOT_FONTPATH =
-
-# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen
-# will generate a graph for each documented class showing the direct and
-# indirect inheritance relations. Setting this tag to YES will force the
-# the CLASS_DIAGRAMS tag to NO.
-
-CLASS_GRAPH = YES
-
-# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen
-# will generate a graph for each documented class showing the direct and
-# indirect implementation dependencies (inheritance, containment, and
-# class references variables) of the class with other documented classes.
-
-COLLABORATION_GRAPH = YES
-
-# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen
-# will generate a graph for groups, showing the direct groups dependencies
-
-GROUP_GRAPHS = YES
-
-# If the UML_LOOK tag is set to YES doxygen will generate inheritance and
-# collaboration diagrams in a style similar to the OMG's Unified Modeling
-# Language.
-
-UML_LOOK = NO
-
-# If set to YES, the inheritance and collaboration graphs will show the
-# relations between templates and their instances.
-
-TEMPLATE_RELATIONS = NO
-
-# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT
-# tags are set to YES then doxygen will generate a graph for each documented
-# file showing the direct and indirect include dependencies of the file with
-# other documented files.
-
-INCLUDE_GRAPH = YES
-
-# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and
-# HAVE_DOT tags are set to YES then doxygen will generate a graph for each
-# documented header file showing the documented files that directly or
-# indirectly include this file.
-
-INCLUDED_BY_GRAPH = YES
-
-# If the CALL_GRAPH and HAVE_DOT options are set to YES then
-# doxygen will generate a call dependency graph for every global function
-# or class method. Note that enabling this option will significantly increase
-# the time of a run. So in most cases it will be better to enable call graphs
-# for selected functions only using the \callgraph command.
-
-CALL_GRAPH = NO
-
-# If the CALLER_GRAPH and HAVE_DOT tags are set to YES then
-# doxygen will generate a caller dependency graph for every global function
-# or class method. Note that enabling this option will significantly increase
-# the time of a run. So in most cases it will be better to enable caller
-# graphs for selected functions only using the \callergraph command.
-
-CALLER_GRAPH = NO
-
-# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen
-# will graphical hierarchy of all classes instead of a textual one.
-
-GRAPHICAL_HIERARCHY = YES
-
-# If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES
-# then doxygen will show the dependencies a directory has on other directories
-# in a graphical way. The dependency relations are determined by the #include
-# relations between the files in the directories.
-
-DIRECTORY_GRAPH = YES
-
-# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images
-# generated by dot. Possible values are png, jpg, or gif
-# If left blank png will be used.
-
-DOT_IMAGE_FORMAT = png
-
-# The tag DOT_PATH can be used to specify the path where the dot tool can be
-# found. If left blank, it is assumed the dot tool can be found in the path.
-
-DOT_PATH =
-
-# The DOTFILE_DIRS tag can be used to specify one or more directories that
-# contain dot files that are included in the documentation (see the
-# \dotfile command).
-
-DOTFILE_DIRS =
-
-# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of
-# nodes that will be shown in the graph. If the number of nodes in a graph
-# becomes larger than this value, doxygen will truncate the graph, which is
-# visualized by representing a node as a red box. Note that doxygen if the
-# number of direct children of the root node in a graph is already larger than
-# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note
-# that the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH.
-
-DOT_GRAPH_MAX_NODES = 50
-
-# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the
-# graphs generated by dot. A depth value of 3 means that only nodes reachable
-# from the root by following a path via at most 3 edges will be shown. Nodes
-# that lay further from the root node will be omitted. Note that setting this
-# option to 1 or 2 may greatly reduce the computation time needed for large
-# code bases. Also note that the size of a graph can be further restricted by
-# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction.
-
-MAX_DOT_GRAPH_DEPTH = 0
-
-# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent
-# background. This is disabled by default, because dot on Windows does not
-# seem to support this out of the box. Warning: Depending on the platform used,
-# enabling this option may lead to badly anti-aliased labels on the edges of
-# a graph (i.e. they become hard to read).
-
-DOT_TRANSPARENT = NO
-
-# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output
-# files in one run (i.e. multiple -o and -T options on the command line). This
-# makes dot run faster, but since only newer versions of dot (>1.8.10)
-# support this, this feature is disabled by default.
-
-DOT_MULTI_TARGETS = NO
-
-# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will
-# generate a legend page explaining the meaning of the various boxes and
-# arrows in the dot generated graphs.
-
-GENERATE_LEGEND = YES
-
-# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will
-# remove the intermediate dot files that are used to generate
-# the various graphs.
-
-DOT_CLEANUP = YES
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/Jamroot
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/Jamroot b/ext/kenlm/Jamroot
deleted file mode 100644
index 6570c23..0000000
--- a/ext/kenlm/Jamroot
+++ /dev/null
@@ -1,68 +0,0 @@
-#BUILDING KENLM
-#
-#DEPENDENCIES
-#Remember that some distributions split packages into development and binary
-#versions. You need the development versions.
-#
-#zlib and Boost >= 1.36.0 are required. Don't have Boost? You can compile the
-#queries part with ./compile_query_only.sh .
-#--with-boost=/path/to/boost uses a Boost install from a custom path.
-#
-#The following optional packages are autodetected:
-#libtcmalloc_minimal from google-perftools
-#bzip2
-#lzma aka xz
-#
-#INSTALLATION
-#--prefix=/path/to/prefix sets the install prefix [default is source root].
-#--bindir=/path/to/prefix/bin sets the bin directory [PREFIX/bin]
-#--libdir=/path/to/prefix/lib sets the lib directory [PREFIX/lib]
-#--includedir=/path/to/prefix/include installs headers.
-# Does not install if missing. No argument defaults to PREFIX/include .
-#--install-scripts=/path/to/scripts copies scripts into a directory.
-# Does not install if missing. No argument defaults to PREFIX/scripts .
-#--git appends the git revision to the prefix directory.
-#
-#BUILD OPTIONS
-# By default, the build is multi-threaded, optimized, and statically linked.
-# Pass these to change the build:
-#
-# threading=single|multi controls threading (default multi)
-#
-# variant=release|debug|profile builds optimized (default), for debug, or for
-# profiling
-#
-# link=static|shared controls preferred linking (default static)
-# --static forces static linking (the default will fall
-# back to shared)
-#
-# debug-symbols=on|off include or exclude (default) debugging
-# information also known as -g
-# --without-libsegfault does not link with libSegFault
-#
-# --max-kenlm-order maximum ngram order that kenlm can process (default 6)
-#
-#CONTROLLING THE BUILD
-#-a to build from scratch
-#-j$NCPUS to compile in parallel
-#--clean to clean
-
-path-constant TOP : . ;
-include $(TOP)/jam-files/sanity.jam ;
-
-boost 103600 ;
-project : requirements $(requirements) <include>. ;
-project : default-build <threading>multi <warnings>on <variant>release <link>static ;
-
-external-lib z ;
-
-build-project lm ;
-build-project util ;
-
-lib kenlm : lm//kenlm ;
-
-install-bin-libs lm//programs util//programs kenlm ;
-install-headers prefix-include : [ glob-tree *.hh : dist include prefix-include ] : . ;
-alias install : prefix-bin prefix-lib prefix-include ;
-explicit headers ;
-explicit install ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/LICENSE
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/LICENSE b/ext/kenlm/LICENSE
deleted file mode 100644
index 6dfbb00..0000000
--- a/ext/kenlm/LICENSE
+++ /dev/null
@@ -1,26 +0,0 @@
-Most of the code here is licensed under the LGPL. There are exceptions that
-have their own licenses, listed below. See comments in those files for more
-details.
-
-util/getopt.* is getopt for Windows
-util/murmur_hash.cc
-util/string_piece.hh and util/string_piece.cc
-util/double-conversion/LICENSE covers util/double-conversion except Jamfile
-util/file.cc contains a modified implementation of mkstemp under the LGPL
-util/integer_to_string.* is BSD
-jam-files/LICENSE_1_0.txt covers jam-files except Jamroot
-
-For the rest:
-
- KenLM is free software: you can redistribute it and/or modify
- it under the terms of the GNU Lesser General Public License as published
- by the Free Software Foundation, either version 2.1 of the License, or
- (at your option) any later version.
-
- KenLM is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public License 2.1
- along with KenLM code. If not, see <http://www.gnu.org/licenses/lgpl-2.1.html>.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/README.md
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/README.md b/ext/kenlm/README.md
deleted file mode 100644
index 1288504..0000000
--- a/ext/kenlm/README.md
+++ /dev/null
@@ -1,96 +0,0 @@
-# kenlm
-
-Language model inference code by Kenneth Heafield (kenlm at kheafield.com)
-
-I do development in master on https://github.com/kpu/kenlm/. Normally, it works, but I do not guarantee it will compile, give correct answers, or generate non-broken binary files. For a more stable release, get http://kheafield.com/code/kenlm.tar.gz .
-
-The website http://kheafield.com/code/kenlm/ has more documentation. If you're a decoder developer, please download the latest version from there instead of copying from another decoder.
-
-## Compiling
-See BUILDING.
-
-## Compiling with your own build system
-If you want to compile with your own build system (Makefile etc) or to use as a library, there are a number of macros you can set on the g++ command line or in util/have.hh .
-
-* `KENLM_MAX_ORDER` is the maximum order that can be loaded. This is done to make state an efficient POD rather than a vector.
-* `HAVE_ICU` If your code links against ICU, define this to disable the internal StringPiece and replace it with ICU's copy of StringPiece, avoiding naming conflicts.
-
-ARPA files can be read in compressed format with these options:
-* `HAVE_ZLIB` Supports gzip. Link with -lz.
-* `HAVE_BZLIB` Supports bzip2. Link with -lbz2.
-* `HAVE_XZLIB` Supports xz. Link with -llzma.
-
-Note that these macros impact only `read_compressed.cc` and `read_compressed_test.cc`. The bjam build system will auto-detect bzip2 and xz support.
-
-## Estimation
-lmplz estimates unpruned language models with modified Kneser-Ney smoothing. After compiling with bjam, run
-```bash
-bin/lmplz -o 5 <text >text.arpa
-```
-The algorithm is on-disk, using an amount of memory that you specify. See http://kheafield.com/code/kenlm/estimation/ for more.
-
-MT Marathon 2012 team members Ivan Pouzyrevsky and Mohammed Mediani contributed to the computation design and early implementation. Jon Clark contributed to the design, clarified points about smoothing, and added logging.
-
-## Filtering
-
-filter takes an ARPA or count file and removes entries that will never be queried. The filter criterion can be corpus-level vocabulary, sentence-level vocabulary, or sentence-level phrases. Run
-```bash
-bin/filter
-```
-and see http://kheafield.com/code/kenlm/filter/ for more documentation.
-
-## Querying
-
-Two data structures are supported: probing and trie. Probing is a probing hash table with keys that are 64-bit hashes of n-grams and floats as values. Trie is a fairly standard trie but with bit-level packing so it uses the minimum number of bits to store word indices and pointers. The trie node entries are sorted by word index. Probing is the fastest and uses the most memory. Trie uses the least memory and a bit slower.
-
-With trie, resident memory is 58% of IRST's smallest version and 21% of SRI's compact version. Simultaneously, trie CPU's use is 81% of IRST's fastest version and 84% of SRI's fast version. KenLM's probing hash table implementation goes even faster at the expense of using more memory. See http://kheafield.com/code/kenlm/benchmark/.
-
-Binary format via mmap is supported. Run `./build_binary` to make one then pass the binary file name to the appropriate Model constructor.
-
-## Platforms
-`murmur_hash.cc` and `bit_packing.hh` perform unaligned reads and writes that make the code architecture-dependent.
-It has been sucessfully tested on x86\_64, x86, and PPC64.
-ARM support is reportedly working, at least on the iphone.
-
-Runs on Linux, OS X, Cygwin, and MinGW.
-
-Hideo Okuma and Tomoyuki Yoshimura from NICT contributed ports to ARM and MinGW.
-
-## Decoder developers
-- I recommend copying the code and distributing it with your decoder. However, please send improvements upstream.
-
-- Omit the lm/filter directory if you do not want the language model filter. Only that and tests depend on Boost.
-
-- Select the macros you want, listed in the previous section.
-
-- There are two build systems: compile.sh and Jamroot+Jamfile. They're pretty simple and are intended to be reimplemented in your build system.
-
-- Use either the interface in `lm/model.hh` or `lm/virtual_interface.hh`. Interface documentation is in comments of `lm/virtual_interface.hh` and `lm/model.hh`.
-
-- There are several possible data structures in `model.hh`. Use `RecognizeBinary` in `binary_format.hh` to determine which one a user has provided. You probably already implement feature functions as an abstract virtual base class with several children. I suggest you co-opt this existing virtual dispatch by templatizing the language model feature implementation on the KenLM model identified by `RecognizeBinary`. This is the strategy used in Moses and cdec.
-
-- See `lm/config.hh` for run-time tuning options.
-
-## Contributors
-Contributions to KenLM are welcome. Please base your contributions on https://github.com/kpu/kenlm and send pull requests (or I might give you commit access). Downstream copies in Moses and cdec are maintained by overwriting them so do not make changes there.
-
-## Python module
-Contributed by Victor Chahuneau.
-
-### Installation
-
-```bash
-pip install https://github.com/kpu/kenlm/archive/master.zip
-```
-
-### Basic Usage
-```python
-import kenlm
-model = kenlm.LanguageModel('lm/test.arpa')
-sentence = 'this is a sentence .'
-print(model.score(sentence))
-```
-
----
-
-The name was Hieu Hoang's idea, not mine.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/bjam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/bjam b/ext/kenlm/bjam
deleted file mode 100755
index 939ea99..0000000
--- a/ext/kenlm/bjam
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/bin/bash
-set -e
-top="$(dirname "$0")"
-if
- bjam="$(which bjam 2>/dev/null)" && #exists
- [ ${#bjam} != 0 ] && #paranoia about which printing nothing then returning true
- ! grep UFIHGUFIHBDJKNCFZXAEVA "${bjam}" </dev/null >/dev/null && #bjam in path isn't this script
- "${bjam}" --sanity-test 2>/dev/null |grep Sane >/dev/null && #The test in jam-files/sanity.jam passes
- (cd "${top}/jam-files/fail" && ! "${bjam}") >/dev/null #Returns non-zero on failure
-then
- #Delegate to system bjam
- exec "${bjam}" "$@"
-fi
-
-if [ ! -x "$top"/jam-files/bjam-1_55 ]; then
- pushd "$top/jam-files/engine"
- ./build.sh
- cp -f bin.*/bjam ../bjam-1_55
- popd
-fi
-
-export BOOST_BUILD_PATH="$top"/jam-files/boost-build
-exec "$top"/jam-files/bjam-1_55 "$@"
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/clean_query_only.sh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/clean_query_only.sh b/ext/kenlm/clean_query_only.sh
deleted file mode 100755
index 2636265..0000000
--- a/ext/kenlm/clean_query_only.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/bash
-rm -rf {lm,util,util/double-conversion}/*.o bin/{query,build_binary}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/cmake/KenLMFunctions.cmake
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/cmake/KenLMFunctions.cmake b/ext/kenlm/cmake/KenLMFunctions.cmake
deleted file mode 100644
index cf400c1..0000000
--- a/ext/kenlm/cmake/KenLMFunctions.cmake
+++ /dev/null
@@ -1,82 +0,0 @@
-# Helper functions used across the CMake build system
-
-include(CMakeParseArguments)
-
-# Adds a bunch of executables to the build, each depending on the specified
-# dependent object files and linking against the specified libraries
-function(AddExes)
- set(multiValueArgs EXES DEPENDS LIBRARIES)
- cmake_parse_arguments(AddExes "" "" "${multiValueArgs}" ${ARGN})
-
- # Iterate through the executable list
- foreach(exe ${AddExes_EXES})
-
- # Compile the executable, linking against the requisite dependent object files
- add_executable(${exe} ${exe}_main.cc ${AddExes_DEPENDS})
-
- # Link the executable against the supplied libraries
- if(AddExes_LIBRARIES)
- target_link_libraries(${exe} ${AddExes_LIBRARIES})
- endif()
-
- # Group executables together
- set_target_properties(${exe} PROPERTIES FOLDER executables)
-
- # End for loop
- endforeach(exe)
-
- # Install the executable files
- install(TARGETS ${AddExes_EXES} DESTINATION bin)
-endfunction()
-
-# Adds a single test to the build, depending on the specified dependent
-# object files, linking against the specified libraries, and with the
-# specified command line arguments
-function(KenLMAddTest)
- cmake_parse_arguments(KenLMAddTest "" "TEST"
- "DEPENDS;LIBRARIES;TEST_ARGS" ${ARGN})
-
- # Compile the executable, linking against the requisite dependent object files
- add_executable(${KenLMAddTest_TEST}
- ${KenLMAddTest_TEST}.cc
- ${KenLMAddTest_DEPENDS})
-
- # Require the following compile flag
- set_target_properties(${KenLMAddTest_TEST} PROPERTIES COMPILE_FLAGS -DBOOST_TEST_DYN_LINK)
-
- if(KenLMAddTest_LIBRARIES)
- target_link_libraries(${KenLMAddTest_TEST} ${KenLMAddTest_LIBRARIES})
- endif()
-
- set(test_params "")
- if(KenLMAddTest_TEST_ARGS)
- set(test_params ${KenLMAddTest_TEST_ARGS})
- endif()
-
- # Specify command arguments for how to run each unit test
- #
- # Assuming that foo was defined via add_executable(foo ...),
- # the syntax $<TARGET_FILE:foo> gives the full path to the executable.
- #
- add_test(NAME ${KenLMAddTest_TEST}
- COMMAND $<TARGET_FILE:${KenLMAddTest_TEST}> ${test_params})
-
- # Group unit tests together
- set_target_properties(${KenLMAddTest_TEST} PROPERTIES FOLDER "unit_tests")
-endfunction()
-
-# Adds a bunch of tests to the build, each depending on the specified
-# dependent object files and linking against the specified libraries
-function(AddTests)
- set(multiValueArgs TESTS DEPENDS LIBRARIES TEST_ARGS)
- cmake_parse_arguments(AddTests "" "" "${multiValueArgs}" ${ARGN})
-
- # Iterate through the Boost tests list
- foreach(test ${KENLM_BOOST_TESTS_LIST})
-
- KenLMAddTest(TEST ${test}
- DEPENDS ${AddTests_DEPENDS}
- LIBRARIES ${AddTests_LIBRARIES}
- TEST_ARGS ${AddTests_TEST_ARGS})
- endforeach(test)
-endfunction()
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/cmake/modules/FindEigen3.cmake
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/cmake/modules/FindEigen3.cmake b/ext/kenlm/cmake/modules/FindEigen3.cmake
deleted file mode 100644
index cea1afe..0000000
--- a/ext/kenlm/cmake/modules/FindEigen3.cmake
+++ /dev/null
@@ -1,90 +0,0 @@
-# - Try to find Eigen3 lib
-#
-# This module supports requiring a minimum version, e.g. you can do
-# find_package(Eigen3 3.1.2)
-# to require version 3.1.2 or newer of Eigen3.
-#
-# Once done this will define
-#
-# EIGEN3_FOUND - system has eigen lib with correct version
-# EIGEN3_INCLUDE_DIR - the eigen include directory
-# EIGEN3_VERSION - eigen version
-#
-# This module reads hints about search locations from
-# the following enviroment variables:
-#
-# EIGEN3_ROOT
-# EIGEN3_ROOT_DIR
-
-# Copyright (c) 2006, 2007 Montel Laurent, <mo...@kde.org>
-# Copyright (c) 2008, 2009 Gael Guennebaud, <g....@free.fr>
-# Copyright (c) 2009 Benoit Jacob <ja...@gmail.com>
-# Redistribution and use is allowed according to the terms of the 2-clause BSD license.
-
-if(NOT Eigen3_FIND_VERSION)
- if(NOT Eigen3_FIND_VERSION_MAJOR)
- set(Eigen3_FIND_VERSION_MAJOR 2)
- endif(NOT Eigen3_FIND_VERSION_MAJOR)
- if(NOT Eigen3_FIND_VERSION_MINOR)
- set(Eigen3_FIND_VERSION_MINOR 91)
- endif(NOT Eigen3_FIND_VERSION_MINOR)
- if(NOT Eigen3_FIND_VERSION_PATCH)
- set(Eigen3_FIND_VERSION_PATCH 0)
- endif(NOT Eigen3_FIND_VERSION_PATCH)
-
- set(Eigen3_FIND_VERSION "${Eigen3_FIND_VERSION_MAJOR}.${Eigen3_FIND_VERSION_MINOR}.${Eigen3_FIND_VERSION_PATCH}")
-endif(NOT Eigen3_FIND_VERSION)
-
-macro(_eigen3_check_version)
- file(READ "${EIGEN3_INCLUDE_DIR}/Eigen/src/Core/util/Macros.h" _eigen3_version_header)
-
- string(REGEX MATCH "define[ \t]+EIGEN_WORLD_VERSION[ \t]+([0-9]+)" _eigen3_world_version_match "${_eigen3_version_header}")
- set(EIGEN3_WORLD_VERSION "${CMAKE_MATCH_1}")
- string(REGEX MATCH "define[ \t]+EIGEN_MAJOR_VERSION[ \t]+([0-9]+)" _eigen3_major_version_match "${_eigen3_version_header}")
- set(EIGEN3_MAJOR_VERSION "${CMAKE_MATCH_1}")
- string(REGEX MATCH "define[ \t]+EIGEN_MINOR_VERSION[ \t]+([0-9]+)" _eigen3_minor_version_match "${_eigen3_version_header}")
- set(EIGEN3_MINOR_VERSION "${CMAKE_MATCH_1}")
-
- set(EIGEN3_VERSION ${EIGEN3_WORLD_VERSION}.${EIGEN3_MAJOR_VERSION}.${EIGEN3_MINOR_VERSION})
- if(${EIGEN3_VERSION} VERSION_LESS ${Eigen3_FIND_VERSION})
- set(EIGEN3_VERSION_OK FALSE)
- else(${EIGEN3_VERSION} VERSION_LESS ${Eigen3_FIND_VERSION})
- set(EIGEN3_VERSION_OK TRUE)
- endif(${EIGEN3_VERSION} VERSION_LESS ${Eigen3_FIND_VERSION})
-
- if(NOT EIGEN3_VERSION_OK)
-
- message(STATUS "Eigen3 version ${EIGEN3_VERSION} found in ${EIGEN3_INCLUDE_DIR}, "
- "but at least version ${Eigen3_FIND_VERSION} is required")
- endif(NOT EIGEN3_VERSION_OK)
-endmacro(_eigen3_check_version)
-
-if (EIGEN3_INCLUDE_DIR)
-
- # in cache already
- _eigen3_check_version()
- set(EIGEN3_FOUND ${EIGEN3_VERSION_OK})
-
-else (EIGEN3_INCLUDE_DIR)
-
- find_path(EIGEN3_INCLUDE_DIR NAMES signature_of_eigen3_matrix_library
- HINTS
- ENV EIGEN3_ROOT
- ENV EIGEN3_ROOT_DIR
- PATHS
- ${CMAKE_INSTALL_PREFIX}/include
- ${KDE4_INCLUDE_DIR}
- PATH_SUFFIXES eigen3 eigen
- )
-
- if(EIGEN3_INCLUDE_DIR)
- _eigen3_check_version()
- endif(EIGEN3_INCLUDE_DIR)
-
- include(FindPackageHandleStandardArgs)
- find_package_handle_standard_args(Eigen3 DEFAULT_MSG EIGEN3_INCLUDE_DIR EIGEN3_VERSION_OK)
-
- mark_as_advanced(EIGEN3_INCLUDE_DIR)
-
-endif(EIGEN3_INCLUDE_DIR)
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/compile_query_only.sh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/compile_query_only.sh b/ext/kenlm/compile_query_only.sh
deleted file mode 100755
index 7a82f49..0000000
--- a/ext/kenlm/compile_query_only.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash
-#This is just an example compilation. You should integrate these files into your build system. Boost jam is provided and preferred.
-
-echo You must use ./bjam if you want language model estimation, filtering, or support for compressed files \(.gz, .bz2, .xz\) 1>&2
-
-rm {lm,util}/*.o 2>/dev/null
-set -e
-
-CXX=${CXX:-g++}
-
-CXXFLAGS+=" -I. -O3 -DNDEBUG -DKENLM_MAX_ORDER=6"
-
-#If this fails for you, consider using bjam.
-if [ ${#NPLM} != 0 ]; then
- CXXFLAGS+=" -DHAVE_NPLM -lneuralLM -L$NPLM/src -I$NPLM/src -lboost_thread-mt -fopenmp"
- ADDED_PATHS="lm/wrappers/*.cc"
-fi
-echo 'Compiling with '$CXX $CXXFLAGS
-
-#Grab all cc files in these directories except those ending in test.cc or main.cc
-objects=""
-for i in util/double-conversion/*.cc util/*.cc lm/*.cc $ADDED_PATHS; do
- if [ "${i%test.cc}" == "$i" ] && [ "${i%main.cc}" == "$i" ]; then
- $CXX $CXXFLAGS -c $i -o ${i%.cc}.o
- objects="$objects ${i%.cc}.o"
- fi
-done
-
-mkdir -p bin
-if [ "$(uname)" != Darwin ]; then
- CXXFLAGS="$CXXFLAGS -lrt"
-fi
-$CXX lm/build_binary_main.cc $objects -o bin/build_binary $CXXFLAGS $LDFLAGS
-$CXX lm/query_main.cc $objects -o bin/query $CXXFLAGS $LDFLAGS
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/LICENSE_1_0.txt
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/LICENSE_1_0.txt b/ext/kenlm/jam-files/LICENSE_1_0.txt
deleted file mode 100644
index 36b7cd9..0000000
--- a/ext/kenlm/jam-files/LICENSE_1_0.txt
+++ /dev/null
@@ -1,23 +0,0 @@
-Boost Software License - Version 1.0 - August 17th, 2003
-
-Permission is hereby granted, free of charge, to any person or organization
-obtaining a copy of the software and accompanying documentation covered by
-this license (the "Software") to use, reproduce, display, distribute,
-execute, and transmit the Software, and to prepare derivative works of the
-Software, and to permit third-parties to whom the Software is furnished to
-do so, all subject to the following:
-
-The copyright notices in the Software and this entire statement, including
-the above license grant, this restriction and the following disclaimer,
-must be included in all copies of the Software, in whole or in part, and
-all derivative works of the Software, unless such copies or derivative
-works are solely in the form of machine-executable object code generated by
-a source language processor.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
-SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
-FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
[09/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/ngram_query.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/ngram_query.hh b/ext/kenlm/lm/ngram_query.hh
deleted file mode 100644
index 4430841..0000000
--- a/ext/kenlm/lm/ngram_query.hh
+++ /dev/null
@@ -1,113 +0,0 @@
-#ifndef LM_NGRAM_QUERY_H
-#define LM_NGRAM_QUERY_H
-
-#include "lm/enumerate_vocab.hh"
-#include "lm/model.hh"
-#include "util/file_stream.hh"
-#include "util/file_piece.hh"
-#include "util/usage.hh"
-
-#include <cstdlib>
-#include <string>
-#include <cmath>
-
-namespace lm {
-namespace ngram {
-
-class QueryPrinter {
- public:
- QueryPrinter(int fd, bool print_word, bool print_line, bool print_summary, bool flush)
- : out_(fd), print_word_(print_word), print_line_(print_line), print_summary_(print_summary), flush_(flush) {}
-
- void Word(StringPiece surface, WordIndex vocab, const FullScoreReturn &ret) {
- if (!print_word_) return;
- out_ << surface << '=' << vocab << ' ' << static_cast<unsigned int>(ret.ngram_length) << ' ' << ret.prob << '\t';
- if (flush_) out_.flush();
- }
-
- void Line(uint64_t oov, float total) {
- if (!print_line_) return;
- out_ << "Total: " << total << " OOV: " << oov << '\n';
- if (flush_) out_.flush();
- }
-
- void Summary(double ppl_including_oov, double ppl_excluding_oov, uint64_t corpus_oov, uint64_t corpus_tokens) {
- if (!print_summary_) return;
- out_ <<
- "Perplexity including OOVs:\t" << ppl_including_oov << "\n"
- "Perplexity excluding OOVs:\t" << ppl_excluding_oov << "\n"
- "OOVs:\t" << corpus_oov << "\n"
- "Tokens:\t" << corpus_tokens << '\n';
- out_.flush();
- }
-
- private:
- util::FileStream out_;
- bool print_word_;
- bool print_line_;
- bool print_summary_;
- bool flush_;
-};
-
-template <class Model, class Printer> void Query(const Model &model, bool sentence_context, Printer &printer) {
- typename Model::State state, out;
- lm::FullScoreReturn ret;
- StringPiece word;
-
- util::FilePiece in(0);
-
- double corpus_total = 0.0;
- double corpus_total_oov_only = 0.0;
- uint64_t corpus_oov = 0;
- uint64_t corpus_tokens = 0;
-
- while (true) {
- state = sentence_context ? model.BeginSentenceState() : model.NullContextState();
- float total = 0.0;
- uint64_t oov = 0;
-
- while (in.ReadWordSameLine(word)) {
- lm::WordIndex vocab = model.GetVocabulary().Index(word);
- ret = model.FullScore(state, vocab, out);
- if (vocab == model.GetVocabulary().NotFound()) {
- ++oov;
- corpus_total_oov_only += ret.prob;
- }
- total += ret.prob;
- printer.Word(word, vocab, ret);
- ++corpus_tokens;
- state = out;
- }
- // If people don't have a newline after their last query, this won't add a </s>.
- // Sue me.
- try {
- UTIL_THROW_IF('\n' != in.get(), util::Exception, "FilePiece is confused.");
- } catch (const util::EndOfFileException &e) { break; }
- if (sentence_context) {
- ret = model.FullScore(state, model.GetVocabulary().EndSentence(), out);
- total += ret.prob;
- ++corpus_tokens;
- printer.Word("</s>", model.GetVocabulary().EndSentence(), ret);
- }
- printer.Line(oov, total);
- corpus_total += total;
- corpus_oov += oov;
- }
- printer.Summary(
- pow(10.0, -(corpus_total / static_cast<double>(corpus_tokens))), // PPL including OOVs
- pow(10.0, -((corpus_total - corpus_total_oov_only) / static_cast<double>(corpus_tokens - corpus_oov))), // PPL excluding OOVs
- corpus_oov,
- corpus_tokens);
-}
-
-template <class Model> void Query(const char *file, const Config &config, bool sentence_context, QueryPrinter &printer) {
- Model model(file, config);
- Query<Model, QueryPrinter>(model, sentence_context, printer);
-}
-
-} // namespace ngram
-} // namespace lm
-
-#endif // LM_NGRAM_QUERY_H
-
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/partial.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/partial.hh b/ext/kenlm/lm/partial.hh
deleted file mode 100644
index 9e4e352..0000000
--- a/ext/kenlm/lm/partial.hh
+++ /dev/null
@@ -1,166 +0,0 @@
-#ifndef LM_PARTIAL_H
-#define LM_PARTIAL_H
-
-#include "lm/return.hh"
-#include "lm/state.hh"
-
-#include <algorithm>
-#include <cassert>
-
-namespace lm {
-namespace ngram {
-
-struct ExtendReturn {
- float adjust;
- bool make_full;
- unsigned char next_use;
-};
-
-template <class Model> ExtendReturn ExtendLoop(
- const Model &model,
- unsigned char seen, const WordIndex *add_rbegin, const WordIndex *add_rend, const float *backoff_start,
- const uint64_t *pointers, const uint64_t *pointers_end,
- uint64_t *&pointers_write,
- float *backoff_write) {
- unsigned char add_length = add_rend - add_rbegin;
-
- float backoff_buf[2][KENLM_MAX_ORDER - 1];
- float *backoff_in = backoff_buf[0], *backoff_out = backoff_buf[1];
- std::copy(backoff_start, backoff_start + add_length, backoff_in);
-
- ExtendReturn value;
- value.make_full = false;
- value.adjust = 0.0;
- value.next_use = add_length;
-
- unsigned char i = 0;
- unsigned char length = pointers_end - pointers;
- // pointers_write is NULL means that the existing left state is full, so we should use completed probabilities.
- if (pointers_write) {
- // Using full context, writing to new left state.
- for (; i < length; ++i) {
- FullScoreReturn ret(model.ExtendLeft(
- add_rbegin, add_rbegin + value.next_use,
- backoff_in,
- pointers[i], i + seen + 1,
- backoff_out,
- value.next_use));
- std::swap(backoff_in, backoff_out);
- if (ret.independent_left) {
- value.adjust += ret.prob;
- value.make_full = true;
- ++i;
- break;
- }
- value.adjust += ret.rest;
- *pointers_write++ = ret.extend_left;
- if (value.next_use != add_length) {
- value.make_full = true;
- ++i;
- break;
- }
- }
- }
- // Using some of the new context.
- for (; i < length && value.next_use; ++i) {
- FullScoreReturn ret(model.ExtendLeft(
- add_rbegin, add_rbegin + value.next_use,
- backoff_in,
- pointers[i], i + seen + 1,
- backoff_out,
- value.next_use));
- std::swap(backoff_in, backoff_out);
- value.adjust += ret.prob;
- }
- float unrest = model.UnRest(pointers + i, pointers_end, i + seen + 1);
- // Using none of the new context.
- value.adjust += unrest;
-
- std::copy(backoff_in, backoff_in + value.next_use, backoff_write);
- return value;
-}
-
-template <class Model> float RevealBefore(const Model &model, const Right &reveal, const unsigned char seen, bool reveal_full, Left &left, Right &right) {
- assert(seen < reveal.length || reveal_full);
- uint64_t *pointers_write = reveal_full ? NULL : left.pointers;
- float backoff_buffer[KENLM_MAX_ORDER - 1];
- ExtendReturn value(ExtendLoop(
- model,
- seen, reveal.words + seen, reveal.words + reveal.length, reveal.backoff + seen,
- left.pointers, left.pointers + left.length,
- pointers_write,
- left.full ? backoff_buffer : (right.backoff + right.length)));
- if (reveal_full) {
- left.length = 0;
- value.make_full = true;
- } else {
- left.length = pointers_write - left.pointers;
- value.make_full |= (left.length == model.Order() - 1);
- }
- if (left.full) {
- for (unsigned char i = 0; i < value.next_use; ++i) value.adjust += backoff_buffer[i];
- } else {
- // If left wasn't full when it came in, put words into right state.
- std::copy(reveal.words + seen, reveal.words + seen + value.next_use, right.words + right.length);
- right.length += value.next_use;
- left.full = value.make_full || (right.length == model.Order() - 1);
- }
- return value.adjust;
-}
-
-template <class Model> float RevealAfter(const Model &model, Left &left, Right &right, const Left &reveal, unsigned char seen) {
- assert(seen < reveal.length || reveal.full);
- uint64_t *pointers_write = left.full ? NULL : (left.pointers + left.length);
- ExtendReturn value(ExtendLoop(
- model,
- seen, right.words, right.words + right.length, right.backoff,
- reveal.pointers + seen, reveal.pointers + reveal.length,
- pointers_write,
- right.backoff));
- if (reveal.full) {
- for (unsigned char i = 0; i < value.next_use; ++i) value.adjust += right.backoff[i];
- right.length = 0;
- value.make_full = true;
- } else {
- right.length = value.next_use;
- value.make_full |= (right.length == model.Order() - 1);
- }
- if (!left.full) {
- left.length = pointers_write - left.pointers;
- left.full = value.make_full || (left.length == model.Order() - 1);
- }
- return value.adjust;
-}
-
-template <class Model> float Subsume(const Model &model, Left &first_left, const Right &first_right, const Left &second_left, Right &second_right, const unsigned int between_length) {
- assert(first_right.length < KENLM_MAX_ORDER);
- assert(second_left.length < KENLM_MAX_ORDER);
- assert(between_length < KENLM_MAX_ORDER - 1);
- uint64_t *pointers_write = first_left.full ? NULL : (first_left.pointers + first_left.length);
- float backoff_buffer[KENLM_MAX_ORDER - 1];
- ExtendReturn value(ExtendLoop(
- model,
- between_length, first_right.words, first_right.words + first_right.length, first_right.backoff,
- second_left.pointers, second_left.pointers + second_left.length,
- pointers_write,
- second_left.full ? backoff_buffer : (second_right.backoff + second_right.length)));
- if (second_left.full) {
- for (unsigned char i = 0; i < value.next_use; ++i) value.adjust += backoff_buffer[i];
- } else {
- std::copy(first_right.words, first_right.words + value.next_use, second_right.words + second_right.length);
- second_right.length += value.next_use;
- value.make_full |= (second_right.length == model.Order() - 1);
- }
- if (!first_left.full) {
- first_left.length = pointers_write - first_left.pointers;
- first_left.full = value.make_full || second_left.full || (first_left.length == model.Order() - 1);
- }
- assert(first_left.length < KENLM_MAX_ORDER);
- assert(second_right.length < KENLM_MAX_ORDER);
- return value.adjust;
-}
-
-} // namespace ngram
-} // namespace lm
-
-#endif // LM_PARTIAL_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/partial_test.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/partial_test.cc b/ext/kenlm/lm/partial_test.cc
deleted file mode 100644
index adb644f..0000000
--- a/ext/kenlm/lm/partial_test.cc
+++ /dev/null
@@ -1,199 +0,0 @@
-#include "lm/partial.hh"
-
-#include "lm/left.hh"
-#include "lm/model.hh"
-#include "util/tokenize_piece.hh"
-
-#define BOOST_TEST_MODULE PartialTest
-#include <boost/test/unit_test.hpp>
-#include <boost/test/floating_point_comparison.hpp>
-
-namespace lm {
-namespace ngram {
-namespace {
-
-const char *TestLocation() {
- if (boost::unit_test::framework::master_test_suite().argc < 2) {
- return "test.arpa";
- }
- return boost::unit_test::framework::master_test_suite().argv[1];
-}
-
-Config SilentConfig() {
- Config config;
- config.arpa_complain = Config::NONE;
- config.messages = NULL;
- return config;
-}
-
-struct ModelFixture {
- ModelFixture() : m(TestLocation(), SilentConfig()) {}
-
- RestProbingModel m;
-};
-
-BOOST_FIXTURE_TEST_SUITE(suite, ModelFixture)
-
-BOOST_AUTO_TEST_CASE(SimpleBefore) {
- Left left;
- left.full = false;
- left.length = 0;
- Right right;
- right.length = 0;
-
- Right reveal;
- reveal.length = 1;
- WordIndex period = m.GetVocabulary().Index(".");
- reveal.words[0] = period;
- reveal.backoff[0] = -0.845098;
-
- BOOST_CHECK_CLOSE(0.0, RevealBefore(m, reveal, 0, false, left, right), 0.001);
- BOOST_CHECK_EQUAL(0, left.length);
- BOOST_CHECK(!left.full);
- BOOST_CHECK_EQUAL(1, right.length);
- BOOST_CHECK_EQUAL(period, right.words[0]);
- BOOST_CHECK_CLOSE(-0.845098, right.backoff[0], 0.001);
-
- WordIndex more = m.GetVocabulary().Index("more");
- reveal.words[1] = more;
- reveal.backoff[1] = -0.4771212;
- reveal.length = 2;
- BOOST_CHECK_CLOSE(0.0, RevealBefore(m, reveal, 1, false, left, right), 0.001);
- BOOST_CHECK_EQUAL(0, left.length);
- BOOST_CHECK(!left.full);
- BOOST_CHECK_EQUAL(2, right.length);
- BOOST_CHECK_EQUAL(period, right.words[0]);
- BOOST_CHECK_EQUAL(more, right.words[1]);
- BOOST_CHECK_CLOSE(-0.845098, right.backoff[0], 0.001);
- BOOST_CHECK_CLOSE(-0.4771212, right.backoff[1], 0.001);
-}
-
-BOOST_AUTO_TEST_CASE(AlsoWouldConsider) {
- WordIndex would = m.GetVocabulary().Index("would");
- WordIndex consider = m.GetVocabulary().Index("consider");
-
- ChartState current;
- current.left.length = 1;
- current.left.pointers[0] = would;
- current.left.full = false;
- current.right.length = 1;
- current.right.words[0] = would;
- current.right.backoff[0] = -0.30103;
-
- Left after;
- after.full = false;
- after.length = 1;
- after.pointers[0] = consider;
-
- // adjustment for would consider
- BOOST_CHECK_CLOSE(-1.687872 - -0.2922095 - 0.30103, RevealAfter(m, current.left, current.right, after, 0), 0.001);
-
- BOOST_CHECK_EQUAL(2, current.left.length);
- BOOST_CHECK_EQUAL(would, current.left.pointers[0]);
- BOOST_CHECK_EQUAL(false, current.left.full);
-
- WordIndex also = m.GetVocabulary().Index("also");
- Right before;
- before.length = 1;
- before.words[0] = also;
- before.backoff[0] = -0.30103;
- // r(would) = -0.2922095 [i would], r(would -> consider) = -1.988902 [b(would) + p(consider)]
- // p(also -> would) = -2, p(also would -> consider) = -3
- BOOST_CHECK_CLOSE(-2 + 0.2922095 -3 + 1.988902, RevealBefore(m, before, 0, false, current.left, current.right), 0.001);
- BOOST_CHECK_EQUAL(0, current.left.length);
- BOOST_CHECK(current.left.full);
- BOOST_CHECK_EQUAL(2, current.right.length);
- BOOST_CHECK_EQUAL(would, current.right.words[0]);
- BOOST_CHECK_EQUAL(also, current.right.words[1]);
-}
-
-BOOST_AUTO_TEST_CASE(EndSentence) {
- WordIndex loin = m.GetVocabulary().Index("loin");
- WordIndex period = m.GetVocabulary().Index(".");
- WordIndex eos = m.GetVocabulary().EndSentence();
-
- ChartState between;
- between.left.length = 1;
- between.left.pointers[0] = eos;
- between.left.full = true;
- between.right.length = 0;
-
- Right before;
- before.words[0] = period;
- before.words[1] = loin;
- before.backoff[0] = -0.845098;
- before.backoff[1] = 0.0;
-
- before.length = 1;
- BOOST_CHECK_CLOSE(-0.0410707, RevealBefore(m, before, 0, true, between.left, between.right), 0.001);
- BOOST_CHECK_EQUAL(0, between.left.length);
-}
-
-float ScoreFragment(const RestProbingModel &model, unsigned int *begin, unsigned int *end, ChartState &out) {
- RuleScore<RestProbingModel> scorer(model, out);
- for (unsigned int *i = begin; i < end; ++i) {
- scorer.Terminal(*i);
- }
- return scorer.Finish();
-}
-
-void CheckAdjustment(const RestProbingModel &model, float expect, const Right &before_in, bool before_full, ChartState between, const Left &after_in) {
- Right before(before_in);
- Left after(after_in);
- after.full = false;
- float got = 0.0;
- for (unsigned int i = 1; i < 5; ++i) {
- if (before_in.length >= i) {
- before.length = i;
- got += RevealBefore(model, before, i - 1, false, between.left, between.right);
- }
- if (after_in.length >= i) {
- after.length = i;
- got += RevealAfter(model, between.left, between.right, after, i - 1);
- }
- }
- if (after_in.full) {
- after.full = true;
- got += RevealAfter(model, between.left, between.right, after, after.length);
- }
- if (before_full) {
- got += RevealBefore(model, before, before.length, true, between.left, between.right);
- }
- // Sometimes they're zero and BOOST_CHECK_CLOSE fails for this.
- BOOST_CHECK(fabs(expect - got) < 0.001);
-}
-
-void FullDivide(const RestProbingModel &model, StringPiece str) {
- std::vector<WordIndex> indices;
- for (util::TokenIter<util::SingleCharacter, true> i(str, ' '); i; ++i) {
- indices.push_back(model.GetVocabulary().Index(*i));
- }
- ChartState full_state;
- float full = ScoreFragment(model, &indices.front(), &indices.back() + 1, full_state);
-
- ChartState before_state;
- before_state.left.full = false;
- RuleScore<RestProbingModel> before_scorer(model, before_state);
- float before_score = 0.0;
- for (unsigned int before = 0; before < indices.size(); ++before) {
- for (unsigned int after = before; after <= indices.size(); ++after) {
- ChartState after_state, between_state;
- float after_score = ScoreFragment(model, &indices.front() + after, &indices.front() + indices.size(), after_state);
- float between_score = ScoreFragment(model, &indices.front() + before, &indices.front() + after, between_state);
- CheckAdjustment(model, full - before_score - after_score - between_score, before_state.right, before_state.left.full, between_state, after_state.left);
- }
- before_scorer.Terminal(indices[before]);
- before_score = before_scorer.Finish();
- }
-}
-
-BOOST_AUTO_TEST_CASE(Strings) {
- FullDivide(m, "also would consider");
- FullDivide(m, "looking on a little more loin . </s>");
- FullDivide(m, "in biarritz watching considering looking . on a little more loin also would consider higher to look good unknown the screening foo bar , unknown however unknown </s>");
-}
-
-BOOST_AUTO_TEST_SUITE_END()
-} // namespace
-} // namespace ngram
-} // namespace lm
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/quantize.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/quantize.cc b/ext/kenlm/lm/quantize.cc
deleted file mode 100644
index 02b5dbc..0000000
--- a/ext/kenlm/lm/quantize.cc
+++ /dev/null
@@ -1,93 +0,0 @@
-/* Quantize into bins of equal size as described in
- * M. Federico and N. Bertoldi. 2006. How many bits are needed
- * to store probabilities for phrase-based translation? In Proc.
- * of the Workshop on Statistical Machine Translation, pages
- * 94–101, New York City, June. Association for Computa-
- * tional Linguistics.
- */
-
-#include "lm/quantize.hh"
-
-#include "lm/binary_format.hh"
-#include "lm/lm_exception.hh"
-#include "util/file.hh"
-
-#include <algorithm>
-#include <numeric>
-
-namespace lm {
-namespace ngram {
-
-namespace {
-
-void MakeBins(std::vector<float> &values, float *centers, uint32_t bins) {
- std::sort(values.begin(), values.end());
- std::vector<float>::const_iterator start = values.begin(), finish;
- for (uint32_t i = 0; i < bins; ++i, ++centers, start = finish) {
- finish = values.begin() + ((values.size() * static_cast<uint64_t>(i + 1)) / bins);
- if (finish == start) {
- // zero length bucket.
- *centers = i ? *(centers - 1) : -std::numeric_limits<float>::infinity();
- } else {
- *centers = std::accumulate(start, finish, 0.0) / static_cast<float>(finish - start);
- }
- }
-}
-
-const char kSeparatelyQuantizeVersion = 2;
-
-} // namespace
-
-void SeparatelyQuantize::UpdateConfigFromBinary(const BinaryFormat &file, uint64_t offset, Config &config) {
- unsigned char buffer[3];
- file.ReadForConfig(buffer, 3, offset);
- char version = buffer[0];
- config.prob_bits = buffer[1];
- config.backoff_bits = buffer[2];
- if (version != kSeparatelyQuantizeVersion) UTIL_THROW(FormatLoadException, "This file has quantization version " << (unsigned)version << " but the code expects version " << (unsigned)kSeparatelyQuantizeVersion);
-}
-
-void SeparatelyQuantize::SetupMemory(void *base, unsigned char order, const Config &config) {
- prob_bits_ = config.prob_bits;
- backoff_bits_ = config.backoff_bits;
- // We need the reserved values.
- if (config.prob_bits == 0) UTIL_THROW(ConfigException, "You can't quantize probability to zero");
- if (config.backoff_bits == 0) UTIL_THROW(ConfigException, "You can't quantize backoff to zero");
- if (config.prob_bits > 25) UTIL_THROW(ConfigException, "For efficiency reasons, quantizing probability supports at most 25 bits. Currently you have requested " << static_cast<unsigned>(config.prob_bits) << " bits.");
- if (config.backoff_bits > 25) UTIL_THROW(ConfigException, "For efficiency reasons, quantizing backoff supports at most 25 bits. Currently you have requested " << static_cast<unsigned>(config.backoff_bits) << " bits.");
- // Reserve 8 byte header for bit counts.
- actual_base_ = static_cast<uint8_t*>(base);
- float *start = reinterpret_cast<float*>(actual_base_ + 8);
- for (unsigned char i = 0; i < order - 2; ++i) {
- tables_[i][0] = Bins(prob_bits_, start);
- start += (1ULL << prob_bits_);
- tables_[i][1] = Bins(backoff_bits_, start);
- start += (1ULL << backoff_bits_);
- }
- longest_ = tables_[order - 2][0] = Bins(prob_bits_, start);
-}
-
-void SeparatelyQuantize::Train(uint8_t order, std::vector<float> &prob, std::vector<float> &backoff) {
- TrainProb(order, prob);
-
- // Backoff
- float *centers = tables_[order - 2][1].Populate();
- *(centers++) = kNoExtensionBackoff;
- *(centers++) = kExtensionBackoff;
- MakeBins(backoff, centers, (1ULL << backoff_bits_) - 2);
-}
-
-void SeparatelyQuantize::TrainProb(uint8_t order, std::vector<float> &prob) {
- float *centers = tables_[order - 2][0].Populate();
- MakeBins(prob, centers, (1ULL << prob_bits_));
-}
-
-void SeparatelyQuantize::FinishedLoading(const Config &config) {
- uint8_t *actual_base = actual_base_;
- *(actual_base++) = kSeparatelyQuantizeVersion; // version
- *(actual_base++) = config.prob_bits;
- *(actual_base++) = config.backoff_bits;
-}
-
-} // namespace ngram
-} // namespace lm
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/quantize.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/quantize.hh b/ext/kenlm/lm/quantize.hh
deleted file mode 100644
index 8500ace..0000000
--- a/ext/kenlm/lm/quantize.hh
+++ /dev/null
@@ -1,233 +0,0 @@
-#ifndef LM_QUANTIZE_H
-#define LM_QUANTIZE_H
-
-#include "lm/blank.hh"
-#include "lm/config.hh"
-#include "lm/max_order.hh"
-#include "lm/model_type.hh"
-#include "util/bit_packing.hh"
-
-#include <algorithm>
-#include <vector>
-
-#include <stdint.h>
-
-#include <iostream>
-
-namespace lm {
-namespace ngram {
-
-struct Config;
-class BinaryFormat;
-
-/* Store values directly and don't quantize. */
-class DontQuantize {
- public:
- static const ModelType kModelTypeAdd = static_cast<ModelType>(0);
- static void UpdateConfigFromBinary(const BinaryFormat &, uint64_t, Config &) {}
- static uint64_t Size(uint8_t /*order*/, const Config &/*config*/) { return 0; }
- static uint8_t MiddleBits(const Config &/*config*/) { return 63; }
- static uint8_t LongestBits(const Config &/*config*/) { return 31; }
-
- class MiddlePointer {
- public:
- MiddlePointer(const DontQuantize & /*quant*/, unsigned char /*order_minus_2*/, util::BitAddress address) : address_(address) {}
-
- MiddlePointer() : address_(NULL, 0) {}
-
- bool Found() const {
- return address_.base != NULL;
- }
-
- float Prob() const {
- return util::ReadNonPositiveFloat31(address_.base, address_.offset);
- }
-
- float Backoff() const {
- return util::ReadFloat32(address_.base, address_.offset + 31);
- }
-
- float Rest() const { return Prob(); }
-
- void Write(float prob, float backoff) {
- util::WriteNonPositiveFloat31(address_.base, address_.offset, prob);
- util::WriteFloat32(address_.base, address_.offset + 31, backoff);
- }
-
- private:
- util::BitAddress address_;
- };
-
- class LongestPointer {
- public:
- explicit LongestPointer(const DontQuantize &/*quant*/, util::BitAddress address) : address_(address) {}
-
- LongestPointer() : address_(NULL, 0) {}
-
- bool Found() const {
- return address_.base != NULL;
- }
-
- float Prob() const {
- return util::ReadNonPositiveFloat31(address_.base, address_.offset);
- }
-
- void Write(float prob) {
- util::WriteNonPositiveFloat31(address_.base, address_.offset, prob);
- }
-
- private:
- util::BitAddress address_;
- };
-
- DontQuantize() {}
-
- void SetupMemory(void * /*start*/, unsigned char /*order*/, const Config & /*config*/) {}
-
- static const bool kTrain = false;
- // These should never be called because kTrain is false.
- void Train(uint8_t /*order*/, std::vector<float> &/*prob*/, std::vector<float> &/*backoff*/) {}
- void TrainProb(uint8_t, std::vector<float> &/*prob*/) {}
-
- void FinishedLoading(const Config &) {}
-};
-
-class SeparatelyQuantize {
- private:
- class Bins {
- public:
- // Sigh C++ default constructor
- Bins() {}
-
- Bins(uint8_t bits, float *begin) : begin_(begin), end_(begin_ + (1ULL << bits)), bits_(bits), mask_((1ULL << bits) - 1) {}
-
- float *Populate() { return begin_; }
-
- uint64_t EncodeProb(float value) const {
- return Encode(value, 0);
- }
-
- uint64_t EncodeBackoff(float value) const {
- if (value == 0.0) {
- return HasExtension(value) ? kExtensionQuant : kNoExtensionQuant;
- }
- return Encode(value, 2);
- }
-
- float Decode(std::size_t off) const { return begin_[off]; }
-
- uint8_t Bits() const { return bits_; }
-
- uint64_t Mask() const { return mask_; }
-
- private:
- uint64_t Encode(float value, size_t reserved) const {
- const float *above = std::lower_bound(static_cast<const float*>(begin_) + reserved, end_, value);
- if (above == begin_ + reserved) return reserved;
- if (above == end_) return end_ - begin_ - 1;
- return above - begin_ - (value - *(above - 1) < *above - value);
- }
-
- float *begin_;
- const float *end_;
- uint8_t bits_;
- uint64_t mask_;
- };
-
- public:
- static const ModelType kModelTypeAdd = kQuantAdd;
-
- static void UpdateConfigFromBinary(const BinaryFormat &file, uint64_t offset, Config &config);
-
- static uint64_t Size(uint8_t order, const Config &config) {
- uint64_t longest_table = (static_cast<uint64_t>(1) << static_cast<uint64_t>(config.prob_bits)) * sizeof(float);
- uint64_t middle_table = (static_cast<uint64_t>(1) << static_cast<uint64_t>(config.backoff_bits)) * sizeof(float) + longest_table;
- // unigrams are currently not quantized so no need for a table.
- return (order - 2) * middle_table + longest_table + /* for the bit counts and alignment padding) */ 8;
- }
-
- static uint8_t MiddleBits(const Config &config) { return config.prob_bits + config.backoff_bits; }
- static uint8_t LongestBits(const Config &config) { return config.prob_bits; }
-
- class MiddlePointer {
- public:
- MiddlePointer(const SeparatelyQuantize &quant, unsigned char order_minus_2, const util::BitAddress &address) : bins_(quant.GetTables(order_minus_2)), address_(address) {}
-
- MiddlePointer() : address_(NULL, 0) {}
-
- bool Found() const { return address_.base != NULL; }
-
- float Prob() const {
- return ProbBins().Decode(util::ReadInt25(address_.base, address_.offset + BackoffBins().Bits(), ProbBins().Bits(), ProbBins().Mask()));
- }
-
- float Backoff() const {
- return BackoffBins().Decode(util::ReadInt25(address_.base, address_.offset, BackoffBins().Bits(), BackoffBins().Mask()));
- }
-
- float Rest() const { return Prob(); }
-
- void Write(float prob, float backoff) const {
- util::WriteInt57(address_.base, address_.offset, ProbBins().Bits() + BackoffBins().Bits(),
- (ProbBins().EncodeProb(prob) << BackoffBins().Bits()) | BackoffBins().EncodeBackoff(backoff));
- }
-
- private:
- const Bins &ProbBins() const { return bins_[0]; }
- const Bins &BackoffBins() const { return bins_[1]; }
- const Bins *bins_;
-
- util::BitAddress address_;
- };
-
- class LongestPointer {
- public:
- LongestPointer(const SeparatelyQuantize &quant, const util::BitAddress &address) : table_(&quant.LongestTable()), address_(address) {}
-
- LongestPointer() : address_(NULL, 0) {}
-
- bool Found() const { return address_.base != NULL; }
-
- void Write(float prob) const {
- util::WriteInt25(address_.base, address_.offset, table_->Bits(), table_->EncodeProb(prob));
- }
-
- float Prob() const {
- return table_->Decode(util::ReadInt25(address_.base, address_.offset, table_->Bits(), table_->Mask()));
- }
-
- private:
- const Bins *table_;
- util::BitAddress address_;
- };
-
- SeparatelyQuantize() {}
-
- void SetupMemory(void *start, unsigned char order, const Config &config);
-
- static const bool kTrain = true;
- // Assumes 0.0 is removed from backoff.
- void Train(uint8_t order, std::vector<float> &prob, std::vector<float> &backoff);
- // Train just probabilities (for longest order).
- void TrainProb(uint8_t order, std::vector<float> &prob);
-
- void FinishedLoading(const Config &config);
-
- const Bins *GetTables(unsigned char order_minus_2) const { return tables_[order_minus_2]; }
-
- const Bins &LongestTable() const { return longest_; }
-
- private:
- Bins tables_[KENLM_MAX_ORDER - 1][2];
-
- Bins longest_;
-
- uint8_t *actual_base_;
-
- uint8_t prob_bits_, backoff_bits_;
-};
-
-} // namespace ngram
-} // namespace lm
-
-#endif // LM_QUANTIZE_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/query_main.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/query_main.cc b/ext/kenlm/lm/query_main.cc
deleted file mode 100644
index 0bd28f7..0000000
--- a/ext/kenlm/lm/query_main.cc
+++ /dev/null
@@ -1,115 +0,0 @@
-#include "lm/ngram_query.hh"
-#include "util/getopt.hh"
-
-#ifdef WITH_NPLM
-#include "lm/wrappers/nplm.hh"
-#endif
-
-#include <stdlib.h>
-
-void Usage(const char *name) {
- std::cerr <<
- "KenLM was compiled with maximum order " << KENLM_MAX_ORDER << ".\n"
- "Usage: " << name << " [-b] [-n] [-w] [-s] lm_file\n"
- "-b: Do not buffer output.\n"
- "-n: Do not wrap the input in <s> and </s>.\n"
- "-v summary|sentence|word: Level of verbosity\n"
- "-l lazy|populate|read|parallel: Load lazily, with populate, or malloc+read\n"
- "The default loading method is populate on Linux and read on others.\n";
- exit(1);
-}
-
-int main(int argc, char *argv[]) {
- if (argc == 1 || (argc == 2 && !strcmp(argv[1], "--help")))
- Usage(argv[0]);
-
- lm::ngram::Config config;
- bool sentence_context = true;
- unsigned int verbosity = 2;
- bool flush = false;
-
- int opt;
- while ((opt = getopt(argc, argv, "bnv:l:")) != -1) {
- switch (opt) {
- case 'b':
- flush = true;
- break;
- case 'n':
- sentence_context = false;
- break;
- case 'v':
- if (!strcmp(optarg, "word") || !strcmp(optarg, "2")) {
- verbosity = 2;
- } else if (!strcmp(optarg, "sentence") || !strcmp(optarg, "1")) {
- verbosity = 1;
- } else if (!strcmp(optarg, "summary") || !strcmp(optarg, "0")) {
- verbosity = 0;
- } else {
- Usage(argv[0]);
- }
- break;
- case 'l':
- if (!strcmp(optarg, "lazy")) {
- config.load_method = util::LAZY;
- } else if (!strcmp(optarg, "populate")) {
- config.load_method = util::POPULATE_OR_READ;
- } else if (!strcmp(optarg, "read")) {
- config.load_method = util::READ;
- } else if (!strcmp(optarg, "parallel")) {
- config.load_method = util::PARALLEL_READ;
- } else {
- Usage(argv[0]);
- }
- break;
- case 'h':
- default:
- Usage(argv[0]);
- }
- }
- if (optind + 1 != argc)
- Usage(argv[0]);
- lm::ngram::QueryPrinter printer(1, verbosity >= 2, verbosity >= 1, true, flush);
- const char *file = argv[optind];
- try {
- using namespace lm::ngram;
- ModelType model_type;
- if (RecognizeBinary(file, model_type)) {
- switch(model_type) {
- case PROBING:
- Query<lm::ngram::ProbingModel>(file, config, sentence_context, printer);
- break;
- case REST_PROBING:
- Query<lm::ngram::RestProbingModel>(file, config, sentence_context, printer);
- break;
- case TRIE:
- Query<TrieModel>(file, config, sentence_context, printer);
- break;
- case QUANT_TRIE:
- Query<QuantTrieModel>(file, config, sentence_context, printer);
- break;
- case ARRAY_TRIE:
- Query<ArrayTrieModel>(file, config, sentence_context, printer);
- break;
- case QUANT_ARRAY_TRIE:
- Query<QuantArrayTrieModel>(file, config, sentence_context, printer);
- break;
- default:
- std::cerr << "Unrecognized kenlm model type " << model_type << std::endl;
- abort();
- }
-#ifdef WITH_NPLM
- } else if (lm::np::Model::Recognize(file)) {
- lm::np::Model model(file);
- Query<lm::np::Model, lm::ngram::QueryPrinter>(model, sentence_context, printer);
- Query<lm::np::Model, lm::ngram::QueryPrinter>(model, sentence_context, printer);
-#endif
- } else {
- Query<ProbingModel>(file, config, sentence_context, printer);
- }
- util::PrintUsage(std::cerr);
- } catch (const std::exception &e) {
- std::cerr << e.what() << std::endl;
- return 1;
- }
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/read_arpa.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/read_arpa.cc b/ext/kenlm/lm/read_arpa.cc
deleted file mode 100644
index dc05a65..0000000
--- a/ext/kenlm/lm/read_arpa.cc
+++ /dev/null
@@ -1,161 +0,0 @@
-#include "lm/read_arpa.hh"
-
-#include "lm/blank.hh"
-#include "util/file.hh"
-
-#include <cmath>
-#include <cstdlib>
-#include <iostream>
-#include <sstream>
-#include <vector>
-
-#include <cctype>
-#include <cstring>
-#include <stdint.h>
-
-#ifdef WIN32
-#include <float.h>
-#endif
-
-namespace lm {
-
-// 1 for '\t', '\n', and ' '. This is stricter than isspace.
-const bool kARPASpaces[256] = {0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
-
-namespace {
-
-bool IsEntirelyWhiteSpace(const StringPiece &line) {
- for (size_t i = 0; i < static_cast<size_t>(line.size()); ++i) {
- if (!isspace(line.data()[i])) return false;
- }
- return true;
-}
-
-const char kBinaryMagic[] = "mmap lm http://kheafield.com/code";
-
-// strtoull isn't portable enough :-(
-uint64_t ReadCount(const std::string &from) {
- std::stringstream stream(from);
- uint64_t ret;
- stream >> ret;
- UTIL_THROW_IF(!stream, FormatLoadException, "Bad count " << from);
- return ret;
-}
-
-} // namespace
-
-void ReadARPACounts(util::FilePiece &in, std::vector<uint64_t> &number) {
- number.clear();
- StringPiece line = in.ReadLine();
- // In general, ARPA files can have arbitrary text before "\data\"
- // But in KenLM, we require such lines to start with "#", so that
- // we can do stricter error checking
- while (IsEntirelyWhiteSpace(line) || starts_with(line, "#")) {
- line = in.ReadLine();
- }
-
- if (line != "\\data\\") {
- if ((line.size() >= 2) && (line.data()[0] == 0x1f) && (static_cast<unsigned char>(line.data()[1]) == 0x8b)) {
- UTIL_THROW(FormatLoadException, "Looks like a gzip file. If this is an ARPA file, pipe " << in.FileName() << " through zcat. If this already in binary format, you need to decompress it because mmap doesn't work on top of gzip.");
- }
- if (static_cast<size_t>(line.size()) >= strlen(kBinaryMagic) && StringPiece(line.data(), strlen(kBinaryMagic)) == kBinaryMagic)
- UTIL_THROW(FormatLoadException, "This looks like a binary file but got sent to the ARPA parser. Did you compress the binary file or pass a binary file where only ARPA files are accepted?");
- UTIL_THROW_IF(line.size() >= 4 && StringPiece(line.data(), 4) == "blmt", FormatLoadException, "This looks like an IRSTLM binary file. Did you forget to pass --text yes to compile-lm?");
- UTIL_THROW_IF(line == "iARPA", FormatLoadException, "This looks like an IRSTLM iARPA file. You need an ARPA file. Run\n compile-lm --text yes " << in.FileName() << " " << in.FileName() << ".arpa\nfirst.");
- UTIL_THROW(FormatLoadException, "first non-empty line was \"" << line << "\" not \\data\\.");
- }
- while (!IsEntirelyWhiteSpace(line = in.ReadLine())) {
- if (line.size() < 6 || strncmp(line.data(), "ngram ", 6)) UTIL_THROW(FormatLoadException, "count line \"" << line << "\"doesn't begin with \"ngram \"");
- // So strtol doesn't go off the end of line.
- std::string remaining(line.data() + 6, line.size() - 6);
- char *end_ptr;
- unsigned int length = std::strtol(remaining.c_str(), &end_ptr, 10);
- if ((end_ptr == remaining.c_str()) || (length - 1 != number.size())) UTIL_THROW(FormatLoadException, "ngram count lengths should be consecutive starting with 1: " << line);
- if (*end_ptr != '=') UTIL_THROW(FormatLoadException, "Expected = immediately following the first number in the count line " << line);
- ++end_ptr;
- number.push_back(ReadCount(end_ptr));
- }
-}
-
-void ReadNGramHeader(util::FilePiece &in, unsigned int length) {
- StringPiece line;
- while (IsEntirelyWhiteSpace(line = in.ReadLine())) {}
- std::stringstream expected;
- expected << '\\' << length << "-grams:";
- if (line != expected.str()) UTIL_THROW(FormatLoadException, "Was expecting n-gram header " << expected.str() << " but got " << line << " instead");
-}
-
-void ReadBackoff(util::FilePiece &in, Prob &/*weights*/) {
- switch (in.get()) {
- case '\t':
- {
- float got = in.ReadFloat();
- if (got != 0.0)
- UTIL_THROW(FormatLoadException, "Non-zero backoff " << got << " provided for an n-gram that should have no backoff");
- }
- break;
- case '\n':
- break;
- default:
- UTIL_THROW(FormatLoadException, "Expected tab or newline for backoff");
- }
-}
-
-void ReadBackoff(util::FilePiece &in, float &backoff) {
- // Always make zero negative.
- // Negative zero means that no (n+1)-gram has this n-gram as context.
- // Therefore the hypothesis state can be shorter. Of course, many n-grams
- // are context for (n+1)-grams. An algorithm in the data structure will go
- // back and set the backoff to positive zero in these cases.
- switch (in.get()) {
- case '\t':
- backoff = in.ReadFloat();
- if (backoff == ngram::kExtensionBackoff) backoff = ngram::kNoExtensionBackoff;
- {
-#if defined(WIN32) && !defined(__MINGW32__)
- int float_class = _fpclass(backoff);
- UTIL_THROW_IF(float_class == _FPCLASS_SNAN || float_class == _FPCLASS_QNAN || float_class == _FPCLASS_NINF || float_class == _FPCLASS_PINF, FormatLoadException, "Bad backoff " << backoff);
-#else
- int float_class = std::fpclassify(backoff);
- UTIL_THROW_IF(float_class == FP_NAN || float_class == FP_INFINITE, FormatLoadException, "Bad backoff " << backoff);
-#endif
- }
- UTIL_THROW_IF(in.get() != '\n', FormatLoadException, "Expected newline after backoff");
- break;
- case '\n':
- backoff = ngram::kNoExtensionBackoff;
- break;
- default:
- UTIL_THROW(FormatLoadException, "Expected tab or newline for backoff");
- }
-}
-
-void ReadEnd(util::FilePiece &in) {
- StringPiece line;
- do {
- line = in.ReadLine();
- } while (IsEntirelyWhiteSpace(line));
- if (line != "\\end\\") UTIL_THROW(FormatLoadException, "Expected \\end\\ but the ARPA file has " << line);
-
- try {
- while (true) {
- line = in.ReadLine();
- if (!IsEntirelyWhiteSpace(line)) UTIL_THROW(FormatLoadException, "Trailing line " << line);
- }
- } catch (const util::EndOfFileException &e) {}
-}
-
-void PositiveProbWarn::Warn(float prob) {
- switch (action_) {
- case THROW_UP:
- UTIL_THROW(FormatLoadException, "Positive log probability " << prob << " in the model. This is a bug in IRSTLM; you can set config.positive_log_probability = SILENT or pass -i to build_binary to substitute 0.0 for the log probability. Error");
- case COMPLAIN:
- std::cerr << "There's a positive log probability " << prob << " in the APRA file, probably because of a bug in IRSTLM. This and subsequent entires will be mapped to 0 log probability." << std::endl;
- action_ = SILENT;
- break;
- case SILENT:
- break;
- }
-}
-
-} // namespace lm
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/read_arpa.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/read_arpa.hh b/ext/kenlm/lm/read_arpa.hh
deleted file mode 100644
index 64eeef3..0000000
--- a/ext/kenlm/lm/read_arpa.hh
+++ /dev/null
@@ -1,95 +0,0 @@
-#ifndef LM_READ_ARPA_H
-#define LM_READ_ARPA_H
-
-#include "lm/lm_exception.hh"
-#include "lm/word_index.hh"
-#include "lm/weights.hh"
-#include "util/file_piece.hh"
-
-#include <cstddef>
-#include <iosfwd>
-#include <vector>
-
-namespace lm {
-
-void ReadARPACounts(util::FilePiece &in, std::vector<uint64_t> &number);
-void ReadNGramHeader(util::FilePiece &in, unsigned int length);
-
-void ReadBackoff(util::FilePiece &in, Prob &weights);
-void ReadBackoff(util::FilePiece &in, float &backoff);
-inline void ReadBackoff(util::FilePiece &in, ProbBackoff &weights) {
- ReadBackoff(in, weights.backoff);
-}
-inline void ReadBackoff(util::FilePiece &in, RestWeights &weights) {
- ReadBackoff(in, weights.backoff);
-}
-
-void ReadEnd(util::FilePiece &in);
-
-extern const bool kARPASpaces[256];
-
-// Positive log probability warning.
-class PositiveProbWarn {
- public:
- PositiveProbWarn() : action_(THROW_UP) {}
-
- explicit PositiveProbWarn(WarningAction action) : action_(action) {}
-
- void Warn(float prob);
-
- private:
- WarningAction action_;
-};
-
-template <class Voc, class Weights> void Read1Gram(util::FilePiece &f, Voc &vocab, Weights *unigrams, PositiveProbWarn &warn) {
- try {
- float prob = f.ReadFloat();
- if (prob > 0.0) {
- warn.Warn(prob);
- prob = 0.0;
- }
- UTIL_THROW_IF(f.get() != '\t', FormatLoadException, "Expected tab after probability");
- WordIndex word = vocab.Insert(f.ReadDelimited(kARPASpaces));
- Weights &w = unigrams[word];
- w.prob = prob;
- ReadBackoff(f, w);
- } catch(util::Exception &e) {
- e << " in the 1-gram at byte " << f.Offset();
- throw;
- }
-}
-
-template <class Voc, class Weights> void Read1Grams(util::FilePiece &f, std::size_t count, Voc &vocab, Weights *unigrams, PositiveProbWarn &warn) {
- ReadNGramHeader(f, 1);
- for (std::size_t i = 0; i < count; ++i) {
- Read1Gram(f, vocab, unigrams, warn);
- }
- vocab.FinishedLoading(unigrams);
-}
-
-// Read ngram, write vocab ids to indices_out.
-template <class Voc, class Weights, class Iterator> void ReadNGram(util::FilePiece &f, const unsigned char n, const Voc &vocab, Iterator indices_out, Weights &weights, PositiveProbWarn &warn) {
- try {
- weights.prob = f.ReadFloat();
- if (weights.prob > 0.0) {
- warn.Warn(weights.prob);
- weights.prob = 0.0;
- }
- for (unsigned char i = 0; i < n; ++i, ++indices_out) {
- StringPiece word(f.ReadDelimited(kARPASpaces));
- WordIndex index = vocab.Index(word);
- *indices_out = index;
- // Check for words mapped to <unk> that are not the string <unk>.
- UTIL_THROW_IF(index == 0 /* mapped to <unk> */ && (word != StringPiece("<unk>", 5)) && (word != StringPiece("<UNK>", 5)),
- FormatLoadException, "Word " << word << " was not seen in the unigrams (which are supposed to list the entire vocabulary) but appears");
- }
- ReadBackoff(f, weights);
- } catch(util::Exception &e) {
- e << " in the " << static_cast<unsigned int>(n) << "-gram at byte " << f.Offset();
- throw;
- }
-}
-
-} // namespace lm
-
-#endif // LM_READ_ARPA_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/return.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/return.hh b/ext/kenlm/lm/return.hh
deleted file mode 100644
index ee1f25e..0000000
--- a/ext/kenlm/lm/return.hh
+++ /dev/null
@@ -1,42 +0,0 @@
-#ifndef LM_RETURN_H
-#define LM_RETURN_H
-
-#include <stdint.h>
-
-namespace lm {
-/* Structure returned by scoring routines. */
-struct FullScoreReturn {
- // log10 probability
- float prob;
-
- /* The length of n-gram matched. Do not use this for recombination.
- * Consider a model containing only the following n-grams:
- * -1 foo
- * -3.14 bar
- * -2.718 baz -5
- * -6 foo bar
- *
- * If you score ``bar'' then ngram_length is 1 and recombination state is the
- * empty string because bar has zero backoff and does not extend to the
- * right.
- * If you score ``foo'' then ngram_length is 1 and recombination state is
- * ``foo''.
- *
- * Ideally, keep output states around and compare them. Failing that,
- * get out_state.ValidLength() and use that length for recombination.
- */
- unsigned char ngram_length;
-
- /* Left extension information. If independent_left is set, then prob is
- * independent of words to the left (up to additional backoff). Otherwise,
- * extend_left indicates how to efficiently extend further to the left.
- */
- bool independent_left;
- uint64_t extend_left; // Defined only if independent_left
-
- // Rest cost for extension to the left.
- float rest;
-};
-
-} // namespace lm
-#endif // LM_RETURN_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/search_hashed.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/search_hashed.cc b/ext/kenlm/lm/search_hashed.cc
deleted file mode 100644
index 7e63e00..0000000
--- a/ext/kenlm/lm/search_hashed.cc
+++ /dev/null
@@ -1,298 +0,0 @@
-#include "lm/search_hashed.hh"
-
-#include "lm/binary_format.hh"
-#include "lm/blank.hh"
-#include "lm/lm_exception.hh"
-#include "lm/model.hh"
-#include "lm/read_arpa.hh"
-#include "lm/value.hh"
-#include "lm/vocab.hh"
-
-#include "util/bit_packing.hh"
-#include "util/file_piece.hh"
-
-#include <string>
-
-namespace lm {
-namespace ngram {
-
-class ProbingModel;
-
-namespace {
-
-/* These are passed to ReadNGrams so that n-grams with zero backoff that appear as context will still be used in state. */
-template <class Middle> class ActivateLowerMiddle {
- public:
- explicit ActivateLowerMiddle(Middle &middle) : modify_(middle) {}
-
- void operator()(const WordIndex *vocab_ids, const unsigned int n) {
- uint64_t hash = static_cast<WordIndex>(vocab_ids[1]);
- for (const WordIndex *i = vocab_ids + 2; i < vocab_ids + n; ++i) {
- hash = detail::CombineWordHash(hash, *i);
- }
- typename Middle::MutableIterator i;
- // TODO: somehow get text of n-gram for this error message.
- if (!modify_.UnsafeMutableFind(hash, i))
- UTIL_THROW(FormatLoadException, "The context of every " << n << "-gram should appear as a " << (n-1) << "-gram");
- SetExtension(i->value.backoff);
- }
-
- private:
- Middle &modify_;
-};
-
-template <class Weights> class ActivateUnigram {
- public:
- explicit ActivateUnigram(Weights *unigram) : modify_(unigram) {}
-
- void operator()(const WordIndex *vocab_ids, const unsigned int /*n*/) {
- // assert(n == 2);
- SetExtension(modify_[vocab_ids[1]].backoff);
- }
-
- private:
- Weights *modify_;
-};
-
-// Find the lower order entry, inserting blanks along the way as necessary.
-template <class Value> void FindLower(
- const std::vector<uint64_t> &keys,
- typename Value::Weights &unigram,
- std::vector<util::ProbingHashTable<typename Value::ProbingEntry, util::IdentityHash> > &middle,
- std::vector<typename Value::Weights *> &between) {
- typename util::ProbingHashTable<typename Value::ProbingEntry, util::IdentityHash>::MutableIterator iter;
- typename Value::ProbingEntry entry;
- // Backoff will always be 0.0. We'll get the probability and rest in another pass.
- entry.value.backoff = kNoExtensionBackoff;
- // Go back and find the longest right-aligned entry, informing it that it extends left. Normally this will match immediately, but sometimes SRI is dumb.
- for (int lower = keys.size() - 2; ; --lower) {
- if (lower == -1) {
- between.push_back(&unigram);
- return;
- }
- entry.key = keys[lower];
- bool found = middle[lower].FindOrInsert(entry, iter);
- between.push_back(&iter->value);
- if (found) return;
- }
-}
-
-// Between usually has single entry, the value to adjust. But sometimes SRI stupidly pruned entries so it has unitialized blank values to be set here.
-template <class Added, class Build> void AdjustLower(
- const Added &added,
- const Build &build,
- std::vector<typename Build::Value::Weights *> &between,
- const unsigned int n,
- const std::vector<WordIndex> &vocab_ids,
- typename Build::Value::Weights *unigrams,
- std::vector<util::ProbingHashTable<typename Build::Value::ProbingEntry, util::IdentityHash> > &middle) {
- typedef typename Build::Value Value;
- if (between.size() == 1) {
- build.MarkExtends(*between.front(), added);
- return;
- }
- typedef util::ProbingHashTable<typename Value::ProbingEntry, util::IdentityHash> Middle;
- float prob = -fabs(between.back()->prob);
- // Order of the n-gram on which probabilities are based.
- unsigned char basis = n - between.size();
- assert(basis != 0);
- typename Build::Value::Weights **change = &between.back();
- // Skip the basis.
- --change;
- if (basis == 1) {
- // Hallucinate a bigram based on a unigram's backoff and a unigram probability.
- float &backoff = unigrams[vocab_ids[1]].backoff;
- SetExtension(backoff);
- prob += backoff;
- (*change)->prob = prob;
- build.SetRest(&*vocab_ids.begin(), 2, **change);
- basis = 2;
- --change;
- }
- uint64_t backoff_hash = static_cast<uint64_t>(vocab_ids[1]);
- for (unsigned char i = 2; i <= basis; ++i) {
- backoff_hash = detail::CombineWordHash(backoff_hash, vocab_ids[i]);
- }
- for (; basis < n - 1; ++basis, --change) {
- typename Middle::MutableIterator gotit;
- if (middle[basis - 2].UnsafeMutableFind(backoff_hash, gotit)) {
- float &backoff = gotit->value.backoff;
- SetExtension(backoff);
- prob += backoff;
- }
- (*change)->prob = prob;
- build.SetRest(&*vocab_ids.begin(), basis + 1, **change);
- backoff_hash = detail::CombineWordHash(backoff_hash, vocab_ids[basis+1]);
- }
-
- typename std::vector<typename Value::Weights *>::const_iterator i(between.begin());
- build.MarkExtends(**i, added);
- const typename Value::Weights *longer = *i;
- // Everything has probability but is not marked as extending.
- for (++i; i != between.end(); ++i) {
- build.MarkExtends(**i, *longer);
- longer = *i;
- }
-}
-
-// Continue marking lower entries even they know that they extend left. This is used for upper/lower bounds.
-template <class Build> void MarkLower(
- const std::vector<uint64_t> &keys,
- const Build &build,
- typename Build::Value::Weights &unigram,
- std::vector<util::ProbingHashTable<typename Build::Value::ProbingEntry, util::IdentityHash> > &middle,
- int start_order,
- const typename Build::Value::Weights &longer) {
- if (start_order == 0) return;
- // Hopefully the compiler will realize that if MarkExtends always returns false, it can simplify this code.
- for (int even_lower = start_order - 2 /* index in middle */; ; --even_lower) {
- if (even_lower == -1) {
- build.MarkExtends(unigram, longer);
- return;
- }
- if (!build.MarkExtends(
- middle[even_lower].UnsafeMutableMustFind(keys[even_lower])->value,
- longer)) return;
- }
-}
-
-template <class Build, class Activate, class Store> void ReadNGrams(
- util::FilePiece &f,
- const unsigned int n,
- const size_t count,
- const ProbingVocabulary &vocab,
- const Build &build,
- typename Build::Value::Weights *unigrams,
- std::vector<util::ProbingHashTable<typename Build::Value::ProbingEntry, util::IdentityHash> > &middle,
- Activate activate,
- Store &store,
- PositiveProbWarn &warn) {
- typedef typename Build::Value Value;
- assert(n >= 2);
- ReadNGramHeader(f, n);
-
- // Both vocab_ids and keys are non-empty because n >= 2.
- // vocab ids of words in reverse order.
- std::vector<WordIndex> vocab_ids(n);
- std::vector<uint64_t> keys(n-1);
- typename Store::Entry entry;
- std::vector<typename Value::Weights *> between;
- for (size_t i = 0; i < count; ++i) {
- ReadNGram(f, n, vocab, vocab_ids.rbegin(), entry.value, warn);
- build.SetRest(&*vocab_ids.begin(), n, entry.value);
-
- keys[0] = detail::CombineWordHash(static_cast<uint64_t>(vocab_ids.front()), vocab_ids[1]);
- for (unsigned int h = 1; h < n - 1; ++h) {
- keys[h] = detail::CombineWordHash(keys[h-1], vocab_ids[h+1]);
- }
- // Initially the sign bit is on, indicating it does not extend left. Most already have this but there might +0.0.
- util::SetSign(entry.value.prob);
- entry.key = keys[n-2];
-
- store.Insert(entry);
- between.clear();
- FindLower<Value>(keys, unigrams[vocab_ids.front()], middle, between);
- AdjustLower<typename Store::Entry::Value, Build>(entry.value, build, between, n, vocab_ids, unigrams, middle);
- if (Build::kMarkEvenLower) MarkLower<Build>(keys, build, unigrams[vocab_ids.front()], middle, n - between.size() - 1, *between.back());
- activate(&*vocab_ids.begin(), n);
- }
-
- store.FinishedInserting();
-}
-
-} // namespace
-namespace detail {
-
-template <class Value> uint8_t *HashedSearch<Value>::SetupMemory(uint8_t *start, const std::vector<uint64_t> &counts, const Config &config) {
- unigram_ = Unigram(start, counts[0]);
- start += Unigram::Size(counts[0]);
- std::size_t allocated;
- middle_.clear();
- for (unsigned int n = 2; n < counts.size(); ++n) {
- allocated = Middle::Size(counts[n - 1], config.probing_multiplier);
- middle_.push_back(Middle(start, allocated));
- start += allocated;
- }
- allocated = Longest::Size(counts.back(), config.probing_multiplier);
- longest_ = Longest(start, allocated);
- start += allocated;
- return start;
-}
-
-/*template <class Value> void HashedSearch<Value>::Relocate(uint8_t *start, const std::vector<uint64_t> &counts, const Config &config) {
- unigram_ = Unigram(start, counts[0]);
- start += Unigram::Size(counts[0]);
- for (unsigned int n = 2; n < counts.size(); ++n) {
- middle[n-2].Relocate(start);
- start += Middle::Size(counts[n - 1], config.probing_multiplier)
- }
- longest_.Relocate(start);
-}*/
-
-template <class Value> void HashedSearch<Value>::InitializeFromARPA(const char * /*file*/, util::FilePiece &f, const std::vector<uint64_t> &counts, const Config &config, ProbingVocabulary &vocab, BinaryFormat &backing) {
- void *vocab_rebase;
- void *search_base = backing.GrowForSearch(Size(counts, config), vocab.UnkCountChangePadding(), vocab_rebase);
- vocab.Relocate(vocab_rebase);
- SetupMemory(reinterpret_cast<uint8_t*>(search_base), counts, config);
-
- PositiveProbWarn warn(config.positive_log_probability);
- Read1Grams(f, counts[0], vocab, unigram_.Raw(), warn);
- CheckSpecials(config, vocab);
- DispatchBuild(f, counts, config, vocab, warn);
-}
-
-template <> void HashedSearch<BackoffValue>::DispatchBuild(util::FilePiece &f, const std::vector<uint64_t> &counts, const Config &config, const ProbingVocabulary &vocab, PositiveProbWarn &warn) {
- NoRestBuild build;
- ApplyBuild(f, counts, vocab, warn, build);
-}
-
-template <> void HashedSearch<RestValue>::DispatchBuild(util::FilePiece &f, const std::vector<uint64_t> &counts, const Config &config, const ProbingVocabulary &vocab, PositiveProbWarn &warn) {
- switch (config.rest_function) {
- case Config::REST_MAX:
- {
- MaxRestBuild build;
- ApplyBuild(f, counts, vocab, warn, build);
- }
- break;
- case Config::REST_LOWER:
- {
- LowerRestBuild<ProbingModel> build(config, counts.size(), vocab);
- ApplyBuild(f, counts, vocab, warn, build);
- }
- break;
- }
-}
-
-template <class Value> template <class Build> void HashedSearch<Value>::ApplyBuild(util::FilePiece &f, const std::vector<uint64_t> &counts, const ProbingVocabulary &vocab, PositiveProbWarn &warn, const Build &build) {
- for (WordIndex i = 0; i < counts[0]; ++i) {
- build.SetRest(&i, (unsigned int)1, unigram_.Raw()[i]);
- }
-
- try {
- if (counts.size() > 2) {
- ReadNGrams<Build, ActivateUnigram<typename Value::Weights>, Middle>(
- f, 2, counts[1], vocab, build, unigram_.Raw(), middle_, ActivateUnigram<typename Value::Weights>(unigram_.Raw()), middle_[0], warn);
- }
- for (unsigned int n = 3; n < counts.size(); ++n) {
- ReadNGrams<Build, ActivateLowerMiddle<Middle>, Middle>(
- f, n, counts[n-1], vocab, build, unigram_.Raw(), middle_, ActivateLowerMiddle<Middle>(middle_[n-3]), middle_[n-2], warn);
- }
- if (counts.size() > 2) {
- ReadNGrams<Build, ActivateLowerMiddle<Middle>, Longest>(
- f, counts.size(), counts[counts.size() - 1], vocab, build, unigram_.Raw(), middle_, ActivateLowerMiddle<Middle>(middle_.back()), longest_, warn);
- } else {
- ReadNGrams<Build, ActivateUnigram<typename Value::Weights>, Longest>(
- f, counts.size(), counts[counts.size() - 1], vocab, build, unigram_.Raw(), middle_, ActivateUnigram<typename Value::Weights>(unigram_.Raw()), longest_, warn);
- }
- } catch (util::ProbingSizeException &e) {
- UTIL_THROW(util::ProbingSizeException, "Avoid pruning n-grams like \"bar baz quux\" when \"foo bar baz quux\" is still in the model. KenLM will work when this pruning happens, but the probing model assumes these events are rare enough that using blank space in the probing hash table will cover all of them. Increase probing_multiplier (-p to build_binary) to add more blank spaces.\n");
- }
- ReadEnd(f);
-}
-
-template class HashedSearch<BackoffValue>;
-template class HashedSearch<RestValue>;
-
-} // namespace detail
-} // namespace ngram
-} // namespace lm
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/search_hashed.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/search_hashed.hh b/ext/kenlm/lm/search_hashed.hh
deleted file mode 100644
index 9dc8445..0000000
--- a/ext/kenlm/lm/search_hashed.hh
+++ /dev/null
@@ -1,192 +0,0 @@
-#ifndef LM_SEARCH_HASHED_H
-#define LM_SEARCH_HASHED_H
-
-#include "lm/model_type.hh"
-#include "lm/config.hh"
-#include "lm/read_arpa.hh"
-#include "lm/return.hh"
-#include "lm/weights.hh"
-
-#include "util/bit_packing.hh"
-#include "util/probing_hash_table.hh"
-
-#include <algorithm>
-#include <iostream>
-#include <vector>
-
-namespace util { class FilePiece; }
-
-namespace lm {
-namespace ngram {
-class BinaryFormat;
-class ProbingVocabulary;
-namespace detail {
-
-inline uint64_t CombineWordHash(uint64_t current, const WordIndex next) {
- uint64_t ret = (current * 8978948897894561157ULL) ^ (static_cast<uint64_t>(1 + next) * 17894857484156487943ULL);
- return ret;
-}
-
-#pragma pack(push)
-#pragma pack(4)
-struct ProbEntry {
- uint64_t key;
- Prob value;
- typedef uint64_t Key;
- typedef Prob Value;
- uint64_t GetKey() const {
- return key;
- }
-};
-
-#pragma pack(pop)
-
-class LongestPointer {
- public:
- explicit LongestPointer(const float &to) : to_(&to) {}
-
- LongestPointer() : to_(NULL) {}
-
- bool Found() const {
- return to_ != NULL;
- }
-
- float Prob() const {
- return *to_;
- }
-
- private:
- const float *to_;
-};
-
-template <class Value> class HashedSearch {
- public:
- typedef uint64_t Node;
-
- typedef typename Value::ProbingProxy UnigramPointer;
- typedef typename Value::ProbingProxy MiddlePointer;
- typedef ::lm::ngram::detail::LongestPointer LongestPointer;
-
- static const ModelType kModelType = Value::kProbingModelType;
- static const bool kDifferentRest = Value::kDifferentRest;
- static const unsigned int kVersion = 0;
-
- // TODO: move probing_multiplier here with next binary file format update.
- static void UpdateConfigFromBinary(const BinaryFormat &, const std::vector<uint64_t> &, uint64_t, Config &) {}
-
- static uint64_t Size(const std::vector<uint64_t> &counts, const Config &config) {
- uint64_t ret = Unigram::Size(counts[0]);
- for (unsigned char n = 1; n < counts.size() - 1; ++n) {
- ret += Middle::Size(counts[n], config.probing_multiplier);
- }
- return ret + Longest::Size(counts.back(), config.probing_multiplier);
- }
-
- uint8_t *SetupMemory(uint8_t *start, const std::vector<uint64_t> &counts, const Config &config);
-
- void InitializeFromARPA(const char *file, util::FilePiece &f, const std::vector<uint64_t> &counts, const Config &config, ProbingVocabulary &vocab, BinaryFormat &backing);
-
- unsigned char Order() const {
- return middle_.size() + 2;
- }
-
- typename Value::Weights &UnknownUnigram() { return unigram_.Unknown(); }
-
- UnigramPointer LookupUnigram(WordIndex word, Node &next, bool &independent_left, uint64_t &extend_left) const {
- extend_left = static_cast<uint64_t>(word);
- next = extend_left;
- UnigramPointer ret(unigram_.Lookup(word));
- independent_left = ret.IndependentLeft();
- return ret;
- }
-
- MiddlePointer Unpack(uint64_t extend_pointer, unsigned char extend_length, Node &node) const {
- node = extend_pointer;
- return MiddlePointer(middle_[extend_length - 2].MustFind(extend_pointer)->value);
- }
-
- MiddlePointer LookupMiddle(unsigned char order_minus_2, WordIndex word, Node &node, bool &independent_left, uint64_t &extend_pointer) const {
- node = CombineWordHash(node, word);
- typename Middle::ConstIterator found;
- if (!middle_[order_minus_2].Find(node, found)) {
- independent_left = true;
- return MiddlePointer();
- }
- extend_pointer = node;
- MiddlePointer ret(found->value);
- independent_left = ret.IndependentLeft();
- return ret;
- }
-
- LongestPointer LookupLongest(WordIndex word, const Node &node) const {
- // Sign bit is always on because longest n-grams do not extend left.
- typename Longest::ConstIterator found;
- if (!longest_.Find(CombineWordHash(node, word), found)) return LongestPointer();
- return LongestPointer(found->value.prob);
- }
-
- // Generate a node without necessarily checking that it actually exists.
- // Optionally return false if it's know to not exist.
- bool FastMakeNode(const WordIndex *begin, const WordIndex *end, Node &node) const {
- assert(begin != end);
- node = static_cast<Node>(*begin);
- for (const WordIndex *i = begin + 1; i < end; ++i) {
- node = CombineWordHash(node, *i);
- }
- return true;
- }
-
- private:
- // Interpret config's rest cost build policy and pass the right template argument to ApplyBuild.
- void DispatchBuild(util::FilePiece &f, const std::vector<uint64_t> &counts, const Config &config, const ProbingVocabulary &vocab, PositiveProbWarn &warn);
-
- template <class Build> void ApplyBuild(util::FilePiece &f, const std::vector<uint64_t> &counts, const ProbingVocabulary &vocab, PositiveProbWarn &warn, const Build &build);
-
- class Unigram {
- public:
- Unigram() {}
-
- Unigram(void *start, uint64_t count) :
- unigram_(static_cast<typename Value::Weights*>(start))
-#ifdef DEBUG
- , count_(count)
-#endif
- {}
-
- static uint64_t Size(uint64_t count) {
- return (count + 1) * sizeof(typename Value::Weights); // +1 for hallucinate <unk>
- }
-
- const typename Value::Weights &Lookup(WordIndex index) const {
-#ifdef DEBUG
- assert(index < count_);
-#endif
- return unigram_[index];
- }
-
- typename Value::Weights &Unknown() { return unigram_[0]; }
-
- // For building.
- typename Value::Weights *Raw() { return unigram_; }
-
- private:
- typename Value::Weights *unigram_;
-#ifdef DEBUG
- uint64_t count_;
-#endif
- };
-
- Unigram unigram_;
-
- typedef util::ProbingHashTable<typename Value::ProbingEntry, util::IdentityHash> Middle;
- std::vector<Middle> middle_;
-
- typedef util::ProbingHashTable<ProbEntry, util::IdentityHash> Longest;
- Longest longest_;
-};
-
-} // namespace detail
-} // namespace ngram
-} // namespace lm
-
-#endif // LM_SEARCH_HASHED_H
[22/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/jambase.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/jambase.c b/ext/kenlm/jam-files/engine/jambase.c
deleted file mode 100644
index b15282b..0000000
--- a/ext/kenlm/jam-files/engine/jambase.c
+++ /dev/null
@@ -1,1691 +0,0 @@
-/* Generated by mkjambase from Jambase */
-char *jambase[] = {
-/* Jambase */
-"if $(NT)\n",
-"{\n",
-"SLASH ?= \\\\ ;\n",
-"}\n",
-"SLASH ?= / ;\n",
-"rule find-to-root ( dir : patterns + )\n",
-"{\n",
-"local globs = [ GLOB $(dir) : $(patterns) ] ;\n",
-"while ! $(globs) && $(dir:P) != $(dir)\n",
-"{\n",
-"dir = $(dir:P) ;\n",
-"globs = [ GLOB $(dir) : $(patterns) ] ;\n",
-"}\n",
-"return $(globs) ;\n",
-"}\n",
-".boost-build-file = ;\n",
-".bootstrap-file = ;\n",
-"BOOST_BUILD_PATH.user-value = $(BOOST_BUILD_PATH) ;\n",
-"if ! $(BOOST_BUILD_PATH) && $(UNIX)\n",
-"{\n",
-"BOOST_BUILD_PATH = /usr/share/boost-build ;\n",
-"}\n",
-"rule _poke ( module-name ? : variables + : value * )\n",
-"{\n",
-"module $(<)\n",
-"{\n",
-"$(>) = $(3) ;\n",
-"}\n",
-"}\n",
-"rule boost-build ( dir ? )\n",
-"{\n",
-"if $(.bootstrap-file)\n",
-"{\n",
-"ECHO \"Error: Illegal attempt to re-bootstrap the build system by invoking\" ;\n",
-"ECHO ;\n",
-"ECHO \" 'boost-build\" $(dir) \";'\" ;\n",
-"ECHO ;\n",
-"EXIT \"Please consult the documentation at 'http://www.boost.org'.\" ;\n",
-"}\n",
-"BOOST_BUILD_PATH = $(dir:R=$(.boost-build-file:D)) $(BOOST_BUILD_PATH) ;\n",
-"_poke .ENVIRON : BOOST_BUILD_PATH : $(BOOST_BUILD_PATH) ;\n",
-"local bootstrap-file = [ GLOB $(BOOST_BUILD_PATH) : bootstrap.jam ] ;\n",
-".bootstrap-file = $(bootstrap-file[1]) ;\n",
-"if ! $(.bootstrap-file)\n",
-"{\n",
-"ECHO \"Unable to load Boost.Build: could not find build system.\" ;\n",
-"ECHO --------------------------------------------------------- ;\n",
-"ECHO \"$(.boost-build-file) attempted to load the build system by invoking\" ;\n",
-"ECHO ;\n",
-"ECHO \" 'boost-build\" $(dir) \";'\" ;\n",
-"ECHO ;\n",
-"ECHO \"but we were unable to find \\\"bootstrap.jam\\\" in the specified directory\" ;\n",
-"ECHO \"or in BOOST_BUILD_PATH (searching \"$(BOOST_BUILD_PATH:J=\", \")\").\" ;\n",
-"ECHO ;\n",
-"EXIT \"Please consult the documentation at 'http://www.boost.org'.\" ;\n",
-"}\n",
-"if [ MATCH .*(--debug-configuration).* : $(ARGV) ]\n",
-"{\n",
-"ECHO \"notice: loading Boost.Build from\"\n",
-"[ NORMALIZE_PATH $(.bootstrap-file:D) ] ;\n",
-"}\n",
-"include $(.bootstrap-file) ;\n",
-"}\n",
-"if [ MATCH .*(b2).* : $(ARGV[1]:BL) ] \n",
-"|| [ MATCH .*(bjam).* : $(ARGV[1]:BL) ]\n",
-"|| $(BOOST_ROOT) # A temporary measure so Jam works with Boost.Build v1.\n",
-"{\n",
-"local search-path = $(BOOST_BUILD_PATH) $(BOOST_ROOT) ;\n",
-"local self = [ SELF_PATH ] ;\n",
-"local boost-build-relative = ../../share/boost-build ;\n",
-"local self-based-path = [ NORMALIZE_PATH $(boost-build-relative:R=$(self)) ] ;\n",
-"local boost-build-files =\n",
-"[ find-to-root [ PWD ] : boost-build.jam ]\n",
-"[ GLOB $(self-based-path) : boost-build.jam ]\n",
-"[ GLOB $(search-path) : boost-build.jam ] ;\n",
-".boost-build-file = $(boost-build-files[1]) ;\n",
-"if ! $(.boost-build-file)\n",
-"{\n",
-"ECHO \"Unable to load Boost.Build: could not find \\\"boost-build.jam\\\"\" ;\n",
-"ECHO --------------------------------------------------------------- ;\n",
-"if ! [ MATCH .*(bjam).* : $(ARGV[1]:BL) ]\n",
-"{\n",
-"ECHO \"BOOST_ROOT must be set, either in the environment, or \" ;\n",
-"ECHO \"on the command-line with -sBOOST_ROOT=..., to the root\" ;\n",
-"ECHO \"of the boost installation.\" ;\n",
-"ECHO ;\n",
-"}\n",
-"ECHO \"Attempted search from\" [ PWD ] \"up to the root\" ;\n",
-"ECHO \"at\" $(self-based-path) ;\n",
-"ECHO \"and in these directories from BOOST_BUILD_PATH and BOOST_ROOT: \"$(search-path:J=\", \")\".\" ;\n",
-"EXIT \"Please consult the documentation at 'http://www.boost.org'.\" ;\n",
-"}\n",
-"if [ MATCH .*(--debug-configuration).* : $(ARGV) ]\n",
-"{\n",
-"ECHO \"notice: found boost-build.jam at\"\n",
-"[ NORMALIZE_PATH $(.boost-build-file) ] ;\n",
-"}\n",
-"include $(.boost-build-file) ;\n",
-"if ! $(.bootstrap-file)\n",
-"{\n",
-"ECHO \"Unable to load Boost.Build\" ;\n",
-"ECHO -------------------------- ;\n",
-"ECHO \"\\\"$(.boost-build-file)\\\" was found by searching from\" [ PWD ] \"up to the root\" ;\n",
-"ECHO \"and in these directories from BOOST_BUILD_PATH and BOOST_ROOT: \"$(search-path:J=\", \")\".\" ;\n",
-"ECHO ;\n",
-"ECHO \"However, it failed to call the \\\"boost-build\\\" rule to indicate\" ;\n",
-"ECHO \"the location of the build system.\" ;\n",
-"ECHO ;\n",
-"EXIT \"Please consult the documentation at 'http://www.boost.org'.\" ;\n",
-"}\n",
-"}\n",
-"else\n",
-"{\n",
-"if $(NT)\n",
-"{\n",
-"local SUPPORTED_TOOLSETS = \"BORLANDC\" \"VC7\" \"VISUALC\" \"VISUALC16\" \"INTELC\" \"WATCOM\"\n",
-"\"MINGW\" \"LCC\" ;\n",
-"TOOLSET = \"\" ;\n",
-"if $(JAM_TOOLSET)\n",
-"{\n",
-"local t ;\n",
-"for t in $(SUPPORTED_TOOLSETS)\n",
-"{\n",
-"$(t) = $($(t):J=\" \") ; # reconstitute paths with spaces in them\n",
-"if $(t) = $(JAM_TOOLSET) { TOOLSET = $(t) ; }\n",
-"}\n",
-"if ! $(TOOLSET)\n",
-"{\n",
-"ECHO \"The JAM_TOOLSET environment variable is defined but its value\" ;\n",
-"ECHO \"is invalid, please use one of the following:\" ;\n",
-"ECHO ;\n",
-"for t in $(SUPPORTED_TOOLSETS) { ECHO \" \" $(t) ; }\n",
-"EXIT ;\n",
-"}\n",
-"}\n",
-"if ! $(TOOLSET)\n",
-"{\n",
-"if $(BCCROOT)\n",
-"{\n",
-"TOOLSET = BORLANDC ;\n",
-"BORLANDC = $(BCCROOT:J=\" \") ;\n",
-"}\n",
-"else if $(MSVC)\n",
-"{\n",
-"TOOLSET = VISUALC16 ;\n",
-"VISUALC16 = $(MSVC:J=\" \") ;\n",
-"}\n",
-"else if $(MSVCNT)\n",
-"{\n",
-"TOOLSET = VISUALC ;\n",
-"VISUALC = $(MSVCNT:J=\" \") ;\n",
-"}\n",
-"else if $(MSVCDir)\n",
-"{\n",
-"TOOLSET = VISUALC ;\n",
-"VISUALC = $(MSVCDir:J=\" \") ;\n",
-"}\n",
-"else if $(MINGW)\n",
-"{\n",
-"TOOLSET = MINGW ;\n",
-"}\n",
-"else\n",
-"{\n",
-"ECHO \"Jam cannot be run because, either:\" ;\n",
-"ECHO \" a. You didn't set BOOST_ROOT to indicate the root of your\" ;\n",
-"ECHO \" Boost installation.\" ;\n",
-"ECHO \" b. You are trying to use stock Jam but didn't indicate which\" ;\n",
-"ECHO \" compilation toolset to use. To do so, follow these simple\" ;\n",
-"ECHO \" instructions:\" ;\n",
-"ECHO ;\n",
-"ECHO \" - define one of the following environment variable, with the\" ;\n",
-"ECHO \" appropriate value according to this list:\" ;\n",
-"ECHO ;\n",
-"ECHO \" Variable Toolset Description\" ;\n",
-"ECHO ;\n",
-"ECHO \" BORLANDC Borland C++ BC++ install path\" ;\n",
-"ECHO \" VISUALC Microsoft Visual C++ VC++ install path\" ;\n",
-"ECHO \" VISUALC16 Microsoft Visual C++ 16 bit VC++ 16 bit install\" ;\n",
-"ECHO \" INTELC Intel C/C++ IC++ install path\" ;\n",
-"ECHO \" WATCOM Watcom C/C++ Watcom install path\" ;\n",
-"ECHO \" MINGW MinGW (gcc) MinGW install path\" ;\n",
-"ECHO \" LCC Win32-LCC LCC-Win32 install path\" ;\n",
-"ECHO ;\n",
-"ECHO \" - define the JAM_TOOLSET environment variable with the *name*\" ;\n",
-"ECHO \" of the toolset variable you want to use.\" ;\n",
-"ECHO ;\n",
-"ECHO \" e.g.: set VISUALC=C:\\\\Visual6\" ;\n",
-"ECHO \" set JAM_TOOLSET=VISUALC\" ;\n",
-"EXIT ;\n",
-"}\n",
-"}\n",
-"CP ?= copy ;\n",
-"RM ?= del /f/q ;\n",
-"SLASH ?= \\\\ ;\n",
-"SUFLIB ?= .lib ;\n",
-"SUFOBJ ?= .obj ;\n",
-"SUFEXE ?= .exe ;\n",
-"if $(TOOLSET) = BORLANDC\n",
-"{\n",
-"ECHO \"Compiler is Borland C++\" ;\n",
-"AR ?= tlib /C /P64 ;\n",
-"CC ?= bcc32 ;\n",
-"CCFLAGS ?= -q -y -d -v -w-par -w-ccc -w-rch -w-pro -w-aus ;\n",
-"C++ ?= bcc32 ;\n",
-"C++FLAGS ?= -q -y -d -v -w-par -w-ccc -w-rch -w-pro -w-aus -P ;\n",
-"LINK ?= $(CC) ;\n",
-"LINKFLAGS ?= $(CCFLAGS) ;\n",
-"STDLIBPATH ?= $(BORLANDC)\\\\lib ;\n",
-"STDHDRS ?= $(BORLANDC)\\\\include ;\n",
-"NOARSCAN ?= true ;\n",
-"}\n",
-"else if $(TOOLSET) = VISUALC16\n",
-"{\n",
-"ECHO \"Compiler is Microsoft Visual C++ 16 bit\" ;\n",
-"AR ?= lib /nologo ;\n",
-"CC ?= cl /nologo ;\n",
-"CCFLAGS ?= /D \\\"WIN\\\" ;\n",
-"C++ ?= $(CC) ;\n",
-"C++FLAGS ?= $(CCFLAGS) ;\n",
-"LINK ?= $(CC) ;\n",
-"LINKFLAGS ?= $(CCFLAGS) ;\n",
-"LINKLIBS ?=\n",
-"\\\"$(VISUALC16)\\\\lib\\\\mlibce.lib\\\"\n",
-"\\\"$(VISUALC16)\\\\lib\\\\oldnames.lib\\\"\n",
-";\n",
-"LINKLIBS ?= ;\n",
-"NOARSCAN ?= true ;\n",
-"OPTIM ?= \"\" ;\n",
-"STDHDRS ?= $(VISUALC16)\\\\include ;\n",
-"UNDEFFLAG ?= \"/u _\" ;\n",
-"}\n",
-"else if $(TOOLSET) = VISUALC\n",
-"{\n",
-"ECHO \"Compiler is Microsoft Visual C++\" ;\n",
-"AR ?= lib ;\n",
-"AS ?= masm386 ;\n",
-"CC ?= cl /nologo ;\n",
-"CCFLAGS ?= \"\" ;\n",
-"C++ ?= $(CC) ;\n",
-"C++FLAGS ?= $(CCFLAGS) ;\n",
-"LINK ?= link /nologo ;\n",
-"LINKFLAGS ?= \"\" ;\n",
-"LINKLIBS ?= \\\"$(VISUALC)\\\\lib\\\\advapi32.lib\\\"\n",
-"\\\"$(VISUALC)\\\\lib\\\\gdi32.lib\\\"\n",
-"\\\"$(VISUALC)\\\\lib\\\\user32.lib\\\"\n",
-"\\\"$(VISUALC)\\\\lib\\\\kernel32.lib\\\" ;\n",
-"OPTIM ?= \"\" ;\n",
-"STDHDRS ?= $(VISUALC)\\\\include ;\n",
-"UNDEFFLAG ?= \"/u _\" ;\n",
-"}\n",
-"else if $(TOOLSET) = VC7\n",
-"{\n",
-"ECHO \"Compiler is Microsoft Visual C++ .NET\" ;\n",
-"AR ?= lib ;\n",
-"AS ?= masm386 ;\n",
-"CC ?= cl /nologo ;\n",
-"CCFLAGS ?= \"\" ;\n",
-"C++ ?= $(CC) ;\n",
-"C++FLAGS ?= $(CCFLAGS) ;\n",
-"LINK ?= link /nologo ;\n",
-"LINKFLAGS ?= \"\" ;\n",
-"LINKLIBS ?= \\\"$(VISUALC)\\\\PlatformSDK\\\\lib\\\\advapi32.lib\\\"\n",
-"\\\"$(VISUALC)\\\\PlatformSDK\\\\lib\\\\gdi32.lib\\\"\n",
-"\\\"$(VISUALC)\\\\PlatformSDK\\\\lib\\\\user32.lib\\\"\n",
-"\\\"$(VISUALC)\\\\PlatformSDK\\\\lib\\\\kernel32.lib\\\" ;\n",
-"OPTIM ?= \"\" ;\n",
-"STDHDRS ?= \\\"$(VISUALC)\\\\include\\\"\n",
-"\\\"$(VISUALC)\\\\PlatformSDK\\\\include\\\" ;\n",
-"UNDEFFLAG ?= \"/u _\" ;\n",
-"}\n",
-"else if $(TOOLSET) = INTELC\n",
-"{\n",
-"ECHO \"Compiler is Intel C/C++\" ;\n",
-"if ! $(VISUALC)\n",
-"{\n",
-"ECHO \"As a special exception, when using the Intel C++ compiler, you need\" ;\n",
-"ECHO \"to define the VISUALC environment variable to indicate the location\" ;\n",
-"ECHO \"of your Visual C++ installation. Aborting..\" ;\n",
-"EXIT ;\n",
-"}\n",
-"AR ?= lib ;\n",
-"AS ?= masm386 ;\n",
-"CC ?= icl /nologo ;\n",
-"CCFLAGS ?= \"\" ;\n",
-"C++ ?= $(CC) ;\n",
-"C++FLAGS ?= $(CCFLAGS) ;\n",
-"LINK ?= link /nologo ;\n",
-"LINKFLAGS ?= \"\" ;\n",
-"LINKLIBS ?= $(VISUALC)\\\\lib\\\\advapi32.lib\n",
-"$(VISUALC)\\\\lib\\\\kernel32.lib\n",
-";\n",
-"OPTIM ?= \"\" ;\n",
-"STDHDRS ?= $(INTELC)\\include $(VISUALC)\\\\include ;\n",
-"UNDEFFLAG ?= \"/u _\" ;\n",
-"}\n",
-"else if $(TOOLSET) = WATCOM\n",
-"{\n",
-"ECHO \"Compiler is Watcom C/C++\" ;\n",
-"AR ?= wlib ;\n",
-"CC ?= wcc386 ;\n",
-"CCFLAGS ?= /zq /DWIN32 /I$(WATCOM)\\\\h ; # zq=quiet\n",
-"C++ ?= wpp386 ;\n",
-"C++FLAGS ?= $(CCFLAGS) ;\n",
-"CP ?= copy ;\n",
-"DOT ?= . ;\n",
-"DOTDOT ?= .. ;\n",
-"LINK ?= wcl386 ;\n",
-"LINKFLAGS ?= /zq ; # zq=quiet\n",
-"LINKLIBS ?= ;\n",
-"MV ?= move ;\n",
-"NOARSCAN ?= true ;\n",
-"OPTIM ?= ;\n",
-"RM ?= del /f ;\n",
-"SLASH ?= \\\\ ;\n",
-"STDHDRS ?= $(WATCOM)\\\\h $(WATCOM)\\\\h\\\\nt ;\n",
-"SUFEXE ?= .exe ;\n",
-"SUFLIB ?= .lib ;\n",
-"SUFOBJ ?= .obj ;\n",
-"UNDEFFLAG ?= \"/u _\" ;\n",
-"}\n",
-"else if $(TOOLSET) = MINGW\n",
-"{\n",
-"ECHO \"Compiler is GCC with Mingw\" ;\n",
-"AR ?= ar -ru ;\n",
-"CC ?= gcc ;\n",
-"CCFLAGS ?= \"\" ;\n",
-"C++ ?= $(CC) ;\n",
-"C++FLAGS ?= $(CCFLAGS) ;\n",
-"LINK ?= $(CC) ;\n",
-"LINKFLAGS ?= \"\" ;\n",
-"LINKLIBS ?= \"\" ;\n",
-"OPTIM ?= ;\n",
-"SUFOBJ = .o ;\n",
-"SUFLIB = .a ;\n",
-"SLASH = / ;\n",
-"}\n",
-"else if $(TOOLSET) = LCC\n",
-"{\n",
-"ECHO \"Compiler is Win32-LCC\" ;\n",
-"AR ?= lcclib ;\n",
-"CC ?= lcc ;\n",
-"CCFLAGS ?= \"\" ;\n",
-"C++ ?= $(CC) ;\n",
-"C++FLAGS ?= $(CCFLAGS) ;\n",
-"LINK ?= lcclnk ;\n",
-"LINKFLAGS ?= \"\" ;\n",
-"LINKLIBS ?= \"\" ;\n",
-"OPTIM ?= ;\n",
-"NOARSCAN = true ;\n",
-"}\n",
-"else\n",
-"{\n",
-"EXIT On NT, set BCCROOT, MSVCNT, MINGW or MSVC to the root of the\n",
-"Borland or Microsoft directories. ;\n",
-"}\n",
-"}\n",
-"else if $(OS2)\n",
-"{\n",
-"local SUPPORTED_TOOLSETS = \"EMX\" \"WATCOM\" ;\n",
-"TOOLSET = \"\" ;\n",
-"if $(JAM_TOOLSET)\n",
-"{\n",
-"local t ;\n",
-"for t in $(SUPPORTED_TOOLSETS)\n",
-"{\n",
-"$(t) = $($(t):J=\" \") ; # reconstitute paths with spaces in them\n",
-"if $(t) = $(JAM_TOOLSET) { TOOLSET = $(t) ; }\n",
-"}\n",
-"if ! $(TOOLSET)\n",
-"{\n",
-"ECHO \"The JAM_TOOLSET environment variable is defined but its value\" ;\n",
-"ECHO \"is invalid, please use one of the following:\" ;\n",
-"ECHO ;\n",
-"for t in $(SUPPORTED_TOOLSETS) { ECHO \" \" $(t) ; }\n",
-"EXIT ;\n",
-"}\n",
-"}\n",
-"if ! $(TOOLSET)\n",
-"{\n",
-"if $(watcom)\n",
-"{\n",
-"WATCOM = $(watcom:J=\" \") ;\n",
-"TOOLSET = WATCOM ;\n",
-"}\n",
-"else\n",
-"{\n",
-"ECHO \"Jam cannot be run because you didn't indicate which compilation toolset\" ;\n",
-"ECHO \"to use. To do so, follow these simple instructions:\" ;\n",
-"ECHO ;\n",
-"ECHO \" - define one of the following environment variable, with the\" ;\n",
-"ECHO \" appropriate value according to this list:\" ;\n",
-"ECHO ;\n",
-"ECHO \" Variable Toolset Description\" ;\n",
-"ECHO ;\n",
-"ECHO \" WATCOM Watcom C/C++ Watcom install path\" ;\n",
-"ECHO \" EMX EMX (gcc) EMX install path\" ;\n",
-"ECHO \" VISUALAGE IBM Visual Age C/C++ VisualAge install path\" ;\n",
-"ECHO ;\n",
-"ECHO \" - define the JAM_TOOLSET environment variable with the *name*\" ;\n",
-"ECHO \" of the toolset variable you want to use.\" ;\n",
-"ECHO ;\n",
-"ECHO \" e.g.: set WATCOM=C:\\WATCOM\" ;\n",
-"ECHO \" set JAM_TOOLSET=WATCOM\" ;\n",
-"ECHO ;\n",
-"EXIT ;\n",
-"}\n",
-"}\n",
-"RM = del /f ;\n",
-"CP = copy ;\n",
-"MV ?= move ;\n",
-"DOT ?= . ;\n",
-"DOTDOT ?= .. ;\n",
-"SUFLIB ?= .lib ;\n",
-"SUFOBJ ?= .obj ;\n",
-"SUFEXE ?= .exe ;\n",
-"if $(TOOLSET) = WATCOM\n",
-"{\n",
-"AR ?= wlib ;\n",
-"BINDIR ?= \\\\os2\\\\apps ;\n",
-"CC ?= wcc386 ;\n",
-"CCFLAGS ?= /zq /DOS2 /I$(WATCOM)\\\\h ; # zq=quiet\n",
-"C++ ?= wpp386 ;\n",
-"C++FLAGS ?= $(CCFLAGS) ;\n",
-"LINK ?= wcl386 ;\n",
-"LINKFLAGS ?= /zq ; # zq=quiet\n",
-"LINKLIBS ?= ;\n",
-"NOARSCAN ?= true ;\n",
-"OPTIM ?= ;\n",
-"SLASH ?= \\\\ ;\n",
-"STDHDRS ?= $(WATCOM)\\\\h ;\n",
-"UNDEFFLAG ?= \"/u _\" ;\n",
-"}\n",
-"else if $(TOOLSET) = EMX\n",
-"{\n",
-"ECHO \"Compiler is GCC-EMX\" ;\n",
-"AR ?= ar -ru ;\n",
-"CC ?= gcc ;\n",
-"CCFLAGS ?= \"\" ;\n",
-"C++ ?= $(CC) ;\n",
-"C++FLAGS ?= $(CCFLAGS) ;\n",
-"LINK ?= $(CC) ;\n",
-"LINKFLAGS ?= \"\" ;\n",
-"LINKLIBS ?= \"\" ;\n",
-"OPTIM ?= ;\n",
-"SUFOBJ = .o ;\n",
-"SUFLIB = .a ;\n",
-"UNDEFFLAG ?= \"-U\" ;\n",
-"SLASH = / ;\n",
-"}\n",
-"else\n",
-"{\n",
-"EXIT \"Sorry, but the $(JAM_TOOLSET) toolset isn't supported for now\" ;\n",
-"}\n",
-"}\n",
-"else if $(VMS)\n",
-"{\n",
-"C++ ?= cxx ;\n",
-"C++FLAGS ?= ;\n",
-"CC ?= cc ;\n",
-"CCFLAGS ?= ;\n",
-"CHMOD ?= set file/prot= ;\n",
-"CP ?= copy/replace ;\n",
-"CRELIB ?= true ;\n",
-"DOT ?= [] ;\n",
-"DOTDOT ?= [-] ;\n",
-"EXEMODE ?= (w:e) ;\n",
-"FILEMODE ?= (w:r) ;\n",
-"HDRS ?= ;\n",
-"LINK ?= link ;\n",
-"LINKFLAGS ?= \"\" ;\n",
-"LINKLIBS ?= ;\n",
-"MKDIR ?= create/dir ;\n",
-"MV ?= rename ;\n",
-"OPTIM ?= \"\" ;\n",
-"RM ?= delete ;\n",
-"RUNVMS ?= mcr ;\n",
-"SHELLMODE ?= (w:er) ;\n",
-"SLASH ?= . ;\n",
-"STDHDRS ?= decc$library_include ;\n",
-"SUFEXE ?= .exe ;\n",
-"SUFLIB ?= .olb ;\n",
-"SUFOBJ ?= .obj ;\n",
-"switch $(OS)\n",
-"{\n",
-"case OPENVMS : CCFLAGS ?= /stand=vaxc ;\n",
-"case VMS : LINKLIBS ?= sys$library:vaxcrtl.olb/lib ;\n",
-"}\n",
-"}\n",
-"else if $(MAC)\n",
-"{\n",
-"local OPT ;\n",
-"CW ?= \"{CW}\" ;\n",
-"MACHDRS ?=\n",
-"\"$(UMACHDRS):Universal:Interfaces:CIncludes\"\n",
-"\"$(CW):MSL:MSL_C:MSL_Common:Include\"\n",
-"\"$(CW):MSL:MSL_C:MSL_MacOS:Include\" ;\n",
-"MACLIBS ?=\n",
-"\"$(CW):MacOS Support:Universal:Libraries:StubLibraries:Interfacelib\"\n",
-"\"$(CW):MacOS Support:Universal:Libraries:StubLibraries:Mathlib\" ;\n",
-"MPWLIBS ?=\n",
-"\"$(CW):MacOS Support:Libraries:Runtime:Runtime PPC:MSL MPWCRuntime.lib\"\n",
-"\"$(CW):MSL:MSL_C:MSL_MacOS:Lib:PPC:MSL C.PPC MPW.Lib\" ;\n",
-"MPWNLLIBS ?=\n",
-"\"$(CW):MacOS Support:Libraries:Runtime:Runtime PPC:MSL MPWCRuntime.lib\"\n",
-"\"$(CW):MSL:MSL_C:MSL_MacOS:Lib:PPC:MSL C.PPC MPW(NL).Lib\" ;\n",
-"SIOUXHDRS ?= ;\n",
-"SIOUXLIBS ?=\n",
-"\"$(CW):MacOS Support:Libraries:Runtime:Runtime PPC:MSL RuntimePPC.lib\"\n",
-"\"$(CW):MSL:MSL_C:MSL_MacOS:Lib:PPC:MSL SIOUX.PPC.Lib\"\n",
-"\"$(CW):MSL:MSL_C:MSL_MacOS:Lib:PPC:MSL C.PPC.Lib\" ;\n",
-"C++ ?= mwcppc ;\n",
-"C++FLAGS ?= -w off -nomapcr ;\n",
-"CC ?= mwcppc ;\n",
-"CCFLAGS ?= -w off -nomapcr ;\n",
-"CP ?= duplicate -y ;\n",
-"DOT ?= \":\" ;\n",
-"DOTDOT ?= \"::\" ;\n",
-"HDRS ?= $(MACHDRS) $(MPWHDRS) ;\n",
-"LINK ?= mwlinkppc ;\n",
-"LINKFLAGS ?= -mpwtool -warn ;\n",
-"LINKLIBS ?= $(MACLIBS) $(MPWLIBS) ;\n",
-"MKDIR ?= newfolder ;\n",
-"MV ?= rename -y ;\n",
-"NOARSCAN ?= true ;\n",
-"OPTIM ?= ;\n",
-"RM ?= delete -y ;\n",
-"SLASH ?= \":\" ;\n",
-"STDHDRS ?= ;\n",
-"SUFLIB ?= .lib ;\n",
-"SUFOBJ ?= .o ;\n",
-"}\n",
-"else if $(OS) = BEOS && $(METROWERKS)\n",
-"{\n",
-"AR ?= mwld -xml -o ;\n",
-"BINDIR ?= /boot/apps ;\n",
-"CC ?= mwcc ;\n",
-"CCFLAGS ?= -nosyspath ;\n",
-"C++ ?= $(CC) ;\n",
-"C++FLAGS ?= -nosyspath ;\n",
-"FORTRAN ?= \"\" ;\n",
-"LIBDIR ?= /boot/develop/libraries ;\n",
-"LINK ?= mwld ;\n",
-"LINKFLAGS ?= \"\" ;\n",
-"MANDIR ?= /boot/documentation/\"Shell Tools\"/HTML ;\n",
-"NOARSCAN ?= true ;\n",
-"STDHDRS ?= /boot/develop/headers/posix ;\n",
-"}\n",
-"else if $(OS) = BEOS\n",
-"{\n",
-"BINDIR ?= /boot/apps ;\n",
-"CC ?= gcc ;\n",
-"C++ ?= $(CC) ;\n",
-"FORTRAN ?= \"\" ;\n",
-"LIBDIR ?= /boot/develop/libraries ;\n",
-"LINK ?= gcc ;\n",
-"LINKLIBS ?= -lnet ;\n",
-"NOARSCAN ?= true ;\n",
-"STDHDRS ?= /boot/develop/headers/posix ;\n",
-"}\n",
-"else if $(UNIX)\n",
-"{\n",
-"switch $(OS)\n",
-"{\n",
-"case AIX :\n",
-"LINKLIBS ?= -lbsd ;\n",
-"case AMIGA :\n",
-"CC ?= gcc ;\n",
-"YACC ?= \"bison -y\" ;\n",
-"case CYGWIN :\n",
-"CC ?= gcc ;\n",
-"CCFLAGS += -D__cygwin__ ;\n",
-"LEX ?= flex ;\n",
-"RANLIB ?= \"\" ;\n",
-"SUFEXE ?= .exe ;\n",
-"YACC ?= \"bison -y\" ;\n",
-"case DGUX :\n",
-"RANLIB ?= \"\" ;\n",
-"RELOCATE ?= true ;\n",
-"case HPUX :\n",
-"YACC = ;\n",
-"CFLAGS += -Ae ;\n",
-"CCFLAGS += -Ae ;\n",
-"RANLIB ?= \"\" ;\n",
-"case INTERIX :\n",
-"CC ?= gcc ;\n",
-"RANLIB ?= \"\" ;\n",
-"case IRIX :\n",
-"RANLIB ?= \"\" ;\n",
-"case MPEIX :\n",
-"CC ?= gcc ;\n",
-"C++ ?= gcc ;\n",
-"CCFLAGS += -D_POSIX_SOURCE ;\n",
-"HDRS += /usr/include ;\n",
-"RANLIB ?= \"\" ;\n",
-"NOARSCAN ?= true ;\n",
-"NOARUPDATE ?= true ;\n",
-"case MVS :\n",
-"RANLIB ?= \"\" ;\n",
-"case NEXT :\n",
-"AR ?= libtool -o ;\n",
-"RANLIB ?= \"\" ;\n",
-"case MACOSX :\n",
-"AR ?= libtool -o ;\n",
-"C++ ?= c++ ;\n",
-"MANDIR ?= /usr/local/share/man ;\n",
-"RANLIB ?= \"\" ;\n",
-"case NCR :\n",
-"RANLIB ?= \"\" ;\n",
-"case PTX :\n",
-"RANLIB ?= \"\" ;\n",
-"case QNX :\n",
-"AR ?= wlib ;\n",
-"CC ?= cc ;\n",
-"CCFLAGS ?= -Q ; # quiet\n",
-"C++ ?= $(CC) ;\n",
-"C++FLAGS ?= -Q ; # quiet\n",
-"LINK ?= $(CC) ;\n",
-"LINKFLAGS ?= -Q ; # quiet\n",
-"NOARSCAN ?= true ;\n",
-"RANLIB ?= \"\" ;\n",
-"case SCO :\n",
-"RANLIB ?= \"\" ;\n",
-"RELOCATE ?= true ;\n",
-"case SINIX :\n",
-"RANLIB ?= \"\" ;\n",
-"case SOLARIS :\n",
-"RANLIB ?= \"\" ;\n",
-"AR ?= \"/usr/ccs/bin/ar ru\" ;\n",
-"case UNICOS :\n",
-"NOARSCAN ?= true ;\n",
-"OPTIM ?= -O0 ;\n",
-"case UNIXWARE :\n",
-"RANLIB ?= \"\" ;\n",
-"RELOCATE ?= true ;\n",
-"}\n",
-"CCFLAGS ?= ;\n",
-"C++FLAGS ?= $(CCFLAGS) ;\n",
-"CHMOD ?= chmod ;\n",
-"CHGRP ?= chgrp ;\n",
-"CHOWN ?= chown ;\n",
-"LEX ?= lex ;\n",
-"LINKFLAGS ?= $(CCFLAGS) ;\n",
-"LINKLIBS ?= ;\n",
-"OPTIM ?= -O ;\n",
-"RANLIB ?= ranlib ;\n",
-"YACC ?= yacc ;\n",
-"YACCFILES ?= y.tab ;\n",
-"YACCFLAGS ?= -d ;\n",
-"}\n",
-"AR ?= ar ru ;\n",
-"AS ?= as ;\n",
-"ASFLAGS ?= ;\n",
-"AWK ?= awk ;\n",
-"BINDIR ?= /usr/local/bin ;\n",
-"C++ ?= cc ;\n",
-"C++FLAGS ?= ;\n",
-"CC ?= cc ;\n",
-"CCFLAGS ?= ;\n",
-"CP ?= cp -f ;\n",
-"CRELIB ?= ;\n",
-"DOT ?= . ;\n",
-"DOTDOT ?= .. ;\n",
-"EXEMODE ?= 711 ;\n",
-"FILEMODE ?= 644 ;\n",
-"FORTRAN ?= f77 ;\n",
-"FORTRANFLAGS ?= ;\n",
-"HDRS ?= ;\n",
-"INSTALLGRIST ?= installed ;\n",
-"JAMFILE ?= Jamfile ;\n",
-"JAMRULES ?= Jamrules ;\n",
-"LEX ?= ;\n",
-"LIBDIR ?= /usr/local/lib ;\n",
-"LINK ?= $(CC) ;\n",
-"LINKFLAGS ?= ;\n",
-"LINKLIBS ?= ;\n",
-"LN ?= ln ;\n",
-"MANDIR ?= /usr/local/man ;\n",
-"MKDIR ?= mkdir ;\n",
-"MV ?= mv -f ;\n",
-"OPTIM ?= ;\n",
-"RCP ?= rcp ;\n",
-"RM ?= rm -f ;\n",
-"RSH ?= rsh ;\n",
-"SED ?= sed ;\n",
-"SHELLHEADER ?= \"#!/bin/sh\" ;\n",
-"SHELLMODE ?= 755 ;\n",
-"SLASH ?= / ;\n",
-"STDHDRS ?= /usr/include ;\n",
-"SUFEXE ?= \"\" ;\n",
-"SUFLIB ?= .a ;\n",
-"SUFOBJ ?= .o ;\n",
-"UNDEFFLAG ?= \"-u _\" ;\n",
-"YACC ?= ;\n",
-"YACCFILES ?= ;\n",
-"YACCFLAGS ?= ;\n",
-"HDRPATTERN =\n",
-"\"^[ ]*#[ ]*include[ ]*[<\\\"]([^\\\">]*)[\\\">].*$\" ;\n",
-"OSFULL = $(OS)$(OSVER)$(OSPLAT) $(OS)$(OSPLAT) $(OS)$(OSVER) $(OS) ;\n",
-"DEPENDS all : shell files lib exe obj ;\n",
-"DEPENDS all shell files lib exe obj : first ;\n",
-"NOTFILE all first shell files lib exe obj dirs clean uninstall ;\n",
-"ALWAYS clean uninstall ;\n",
-"rule As\n",
-"{\n",
-"DEPENDS $(<) : $(>) ;\n",
-"ASFLAGS on $(<) += $(ASFLAGS) $(SUBDIRASFLAGS) ;\n",
-"}\n",
-"rule Bulk\n",
-"{\n",
-"local i ;\n",
-"for i in $(>)\n",
-"{\n",
-"File $(i:D=$(<)) : $(i) ;\n",
-"}\n",
-"}\n",
-"rule Cc\n",
-"{\n",
-"local _h ;\n",
-"DEPENDS $(<) : $(>) ;\n",
-"CCFLAGS on $(<) += $(CCFLAGS) $(SUBDIRCCFLAGS) ;\n",
-"if $(RELOCATE)\n",
-"{\n",
-"CcMv $(<) : $(>) ;\n",
-"}\n",
-"_h = $(SEARCH_SOURCE) $(HDRS) $(SUBDIRHDRS) ;\n",
-"if $(VMS) && $(_h)\n",
-"{\n",
-"SLASHINC on $(<) = \"/inc=(\" $(_h[1]) ,$(_h[2-]) \")\" ;\n",
-"}\n",
-"else if $(MAC) && $(_h)\n",
-"{\n",
-"local _i _j ;\n",
-"_j = $(_h[1]) ;\n",
-"for _i in $(_h[2-])\n",
-"{\n",
-"_j = $(_j),$(_i) ;\n",
-"}\n",
-"MACINC on $(<) = \\\"$(_j)\\\" ;\n",
-"}\n",
-"}\n",
-"rule C++\n",
-"{\n",
-"local _h ;\n",
-"DEPENDS $(<) : $(>) ;\n",
-"C++FLAGS on $(<) += $(C++FLAGS) $(SUBDIRC++FLAGS) ;\n",
-"if $(RELOCATE)\n",
-"{\n",
-"CcMv $(<) : $(>) ;\n",
-"}\n",
-"_h = $(SEARCH_SOURCE) $(HDRS) $(SUBDIRHDRS) ;\n",
-"if $(VMS) && $(_h)\n",
-"{\n",
-"SLASHINC on $(<) = \"/inc=(\" $(_h[1]) ,$(_h[2-]) \")\" ;\n",
-"}\n",
-"else if $(MAC) && $(_h)\n",
-"{\n",
-"local _i _j ;\n",
-"_j = $(_h[1]) ;\n",
-"for _i in $(_h[2-])\n",
-"{\n",
-"_j = $(_j),$(_i) ;\n",
-"}\n",
-"MACINC on $(<) = \\\"$(_j)\\\" ;\n",
-"}\n",
-"}\n",
-"rule Chmod\n",
-"{\n",
-"if $(CHMOD) { Chmod1 $(<) ; }\n",
-"}\n",
-"rule File\n",
-"{\n",
-"DEPENDS files : $(<) ;\n",
-"DEPENDS $(<) : $(>) ;\n",
-"SEARCH on $(>) = $(SEARCH_SOURCE) ;\n",
-"MODE on $(<) = $(FILEMODE) ;\n",
-"Chmod $(<) ;\n",
-"}\n",
-"rule Fortran\n",
-"{\n",
-"DEPENDS $(<) : $(>) ;\n",
-"}\n",
-"rule GenFile\n",
-"{\n",
-"local _t = [ FGristSourceFiles $(<) ] ;\n",
-"local _s = [ FAppendSuffix $(>[1]) : $(SUFEXE) ] ;\n",
-"Depends $(_t) : $(_s) $(>[2-]) ;\n",
-"GenFile1 $(_t) : $(_s) $(>[2-]) ;\n",
-"Clean clean : $(_t) ;\n",
-"}\n",
-"rule GenFile1\n",
-"{\n",
-"MakeLocate $(<) : $(LOCATE_SOURCE) ;\n",
-"SEARCH on $(>) = $(SEARCH_SOURCE) ;\n",
-"}\n",
-"rule HardLink\n",
-"{\n",
-"DEPENDS files : $(<) ;\n",
-"DEPENDS $(<) : $(>) ;\n",
-"SEARCH on $(>) = $(SEARCH_SOURCE) ;\n",
-"}\n",
-"rule HdrMacroFile\n",
-"{\n",
-"HDRMACRO $(<) ;\n",
-"}\n",
-"rule HdrRule\n",
-"{\n",
-"local s ;\n",
-"if $(HDRGRIST)\n",
-"{\n",
-"s = $(>:G=$(HDRGRIST)) ;\n",
-"} else {\n",
-"s = $(>) ;\n",
-"}\n",
-"INCLUDES $(<) : $(s) ;\n",
-"SEARCH on $(s) = $(HDRSEARCH) ;\n",
-"NOCARE $(s) ;\n",
-"HDRSEARCH on $(s) = $(HDRSEARCH) ;\n",
-"HDRSCAN on $(s) = $(HDRSCAN) ;\n",
-"HDRRULE on $(s) = $(HDRRULE) ;\n",
-"HDRGRIST on $(s) = $(HDRGRIST) ;\n",
-"}\n",
-"rule InstallInto\n",
-"{\n",
-"local i t ;\n",
-"t = $(>:G=$(INSTALLGRIST)) ;\n",
-"Depends install : $(t) ;\n",
-"Clean uninstall : $(t) ;\n",
-"SEARCH on $(>) = $(SEARCH_SOURCE) ;\n",
-"MakeLocate $(t) : $(<) ;\n",
-"for i in $(>)\n",
-"{\n",
-"local tt = $(i:G=$(INSTALLGRIST)) ;\n",
-"Depends $(tt) : $(i) ;\n",
-"Install $(tt) : $(i) ;\n",
-"Chmod $(tt) ;\n",
-"if $(OWNER) && $(CHOWN)\n",
-"{\n",
-"Chown $(tt) ;\n",
-"OWNER on $(tt) = $(OWNER) ;\n",
-"}\n",
-"if $(GROUP) && $(CHGRP)\n",
-"{\n",
-"Chgrp $(tt) ;\n",
-"GROUP on $(tt) = $(GROUP) ;\n",
-"}\n",
-"}\n",
-"}\n",
-"rule InstallBin\n",
-"{\n",
-"local _t = [ FAppendSuffix $(>) : $(SUFEXE) ] ;\n",
-"InstallInto $(<) : $(_t) ;\n",
-"MODE on $(_t:G=installed) = $(EXEMODE) ;\n",
-"}\n",
-"rule InstallFile\n",
-"{\n",
-"InstallInto $(<) : $(>) ;\n",
-"MODE on $(>:G=installed) = $(FILEMODE) ;\n",
-"}\n",
-"rule InstallLib\n",
-"{\n",
-"InstallInto $(<) : $(>) ;\n",
-"MODE on $(>:G=installed) = $(FILEMODE) ;\n",
-"}\n",
-"rule InstallMan\n",
-"{\n",
-"local i s d ;\n",
-"for i in $(>)\n",
-"{\n",
-"switch $(i:S)\n",
-"{\n",
-"case .1 : s = 1 ; case .2 : s = 2 ; case .3 : s = 3 ;\n",
-"case .4 : s = 4 ; case .5 : s = 5 ; case .6 : s = 6 ;\n",
-"case .7 : s = 7 ; case .8 : s = 8 ; case .l : s = l ;\n",
-"case .n : s = n ; case .man : s = 1 ;\n",
-"}\n",
-"d = man$(s) ;\n",
-"InstallInto $(d:R=$(<)) : $(i) ;\n",
-"}\n",
-"MODE on $(>:G=installed) = $(FILEMODE) ;\n",
-"}\n",
-"rule InstallShell\n",
-"{\n",
-"InstallInto $(<) : $(>) ;\n",
-"MODE on $(>:G=installed) = $(SHELLMODE) ;\n",
-"}\n",
-"rule Lex\n",
-"{\n",
-"LexMv $(<) : $(>) ;\n",
-"DEPENDS $(<) : $(>) ;\n",
-"MakeLocate $(<) : $(LOCATE_SOURCE) ;\n",
-"Clean clean : $(<) ;\n",
-"}\n",
-"rule Library\n",
-"{\n",
-"LibraryFromObjects $(<) : $(>:S=$(SUFOBJ)) ;\n",
-"Objects $(>) ;\n",
-"}\n",
-"rule LibraryFromObjects\n",
-"{\n",
-"local _i _l _s ;\n",
-"_s = [ FGristFiles $(>) ] ;\n",
-"_l = $(<:S=$(SUFLIB)) ;\n",
-"if $(KEEPOBJS)\n",
-"{\n",
-"DEPENDS obj : $(_s) ;\n",
-"}\n",
-"else\n",
-"{\n",
-"DEPENDS lib : $(_l) ;\n",
-"}\n",
-"if ! $(_l:D)\n",
-"{\n",
-"MakeLocate $(_l) $(_l)($(_s:BS)) : $(LOCATE_TARGET) ;\n",
-"}\n",
-"if $(NOARSCAN)\n",
-"{\n",
-"DEPENDS $(_l) : $(_s) ;\n",
-"}\n",
-"else\n",
-"{\n",
-"DEPENDS $(_l) : $(_l)($(_s:BS)) ;\n",
-"for _i in $(_s)\n",
-"{\n",
-"DEPENDS $(_l)($(_i:BS)) : $(_i) ;\n",
-"}\n",
-"}\n",
-"Clean clean : $(_l) ;\n",
-"if $(CRELIB) { CreLib $(_l) : $(_s[1]) ; }\n",
-"Archive $(_l) : $(_s) ;\n",
-"if $(RANLIB) { Ranlib $(_l) ; }\n",
-"if ! ( $(NOARSCAN) || $(KEEPOBJS) ) { RmTemps $(_l) : $(_s) ; }\n",
-"}\n",
-"rule Link\n",
-"{\n",
-"MODE on $(<) = $(EXEMODE) ;\n",
-"Chmod $(<) ;\n",
-"}\n",
-"rule LinkLibraries\n",
-"{\n",
-"local _t = [ FAppendSuffix $(<) : $(SUFEXE) ] ;\n",
-"DEPENDS $(_t) : $(>:S=$(SUFLIB)) ;\n",
-"NEEDLIBS on $(_t) += $(>:S=$(SUFLIB)) ;\n",
-"}\n",
-"rule Main\n",
-"{\n",
-"MainFromObjects $(<) : $(>:S=$(SUFOBJ)) ;\n",
-"Objects $(>) ;\n",
-"}\n",
-"rule MainFromObjects\n",
-"{\n",
-"local _s _t ;\n",
-"_s = [ FGristFiles $(>) ] ;\n",
-"_t = [ FAppendSuffix $(<) : $(SUFEXE) ] ;\n",
-"if $(_t) != $(<)\n",
-"{\n",
-"DEPENDS $(<) : $(_t) ;\n",
-"NOTFILE $(<) ;\n",
-"}\n",
-"DEPENDS exe : $(_t) ;\n",
-"DEPENDS $(_t) : $(_s) ;\n",
-"MakeLocate $(_t) : $(LOCATE_TARGET) ;\n",
-"Clean clean : $(_t) ;\n",
-"Link $(_t) : $(_s) ;\n",
-"}\n",
-"rule MakeLocate\n",
-"{\n",
-"if $(>)\n",
-"{\n",
-"LOCATE on $(<) = $(>) ;\n",
-"Depends $(<) : $(>[1]) ;\n",
-"MkDir $(>[1]) ;\n",
-"}\n",
-"}\n",
-"rule MkDir\n",
-"{\n",
-"NOUPDATE $(<) ;\n",
-"if $(<) != $(DOT) && ! $($(<)-mkdir)\n",
-"{\n",
-"local s ;\n",
-"$(<)-mkdir = true ;\n",
-"MkDir1 $(<) ;\n",
-"Depends dirs : $(<) ;\n",
-"s = $(<:P) ;\n",
-"if $(NT)\n",
-"{\n",
-"switch $(s)\n",
-"{\n",
-"case *: : s = ;\n",
-"case *:\\\\ : s = ;\n",
-"}\n",
-"}\n",
-"if $(s) && $(s) != $(<)\n",
-"{\n",
-"Depends $(<) : $(s) ;\n",
-"MkDir $(s) ;\n",
-"}\n",
-"else if $(s)\n",
-"{\n",
-"NOTFILE $(s) ;\n",
-"}\n",
-"}\n",
-"}\n",
-"rule Object\n",
-"{\n",
-"local h ;\n",
-"Clean clean : $(<) ;\n",
-"MakeLocate $(<) : $(LOCATE_TARGET) ;\n",
-"SEARCH on $(>) = $(SEARCH_SOURCE) ;\n",
-"HDRS on $(<) = $(SEARCH_SOURCE) $(HDRS) $(SUBDIRHDRS) ;\n",
-"if $(SEARCH_SOURCE)\n",
-"{\n",
-"h = $(SEARCH_SOURCE) ;\n",
-"}\n",
-"else\n",
-"{\n",
-"h = \"\" ;\n",
-"}\n",
-"HDRRULE on $(>) = HdrRule ;\n",
-"HDRSCAN on $(>) = $(HDRPATTERN) ;\n",
-"HDRSEARCH on $(>) = $(HDRS) $(SUBDIRHDRS) $(h) $(STDHDRS) ;\n",
-"HDRGRIST on $(>) = $(HDRGRIST) ;\n",
-"switch $(>:S)\n",
-"{\n",
-"case .asm : As $(<) : $(>) ;\n",
-"case .c : Cc $(<) : $(>) ;\n",
-"case .C : C++ $(<) : $(>) ;\n",
-"case .cc : C++ $(<) : $(>) ;\n",
-"case .cpp : C++ $(<) : $(>) ;\n",
-"case .f : Fortran $(<) : $(>) ;\n",
-"case .l : Cc $(<) : $(<:S=.c) ;\n",
-"Lex $(<:S=.c) : $(>) ;\n",
-"case .s : As $(<) : $(>) ;\n",
-"case .y : Cc $(<) : $(<:S=.c) ;\n",
-"Yacc $(<:S=.c) : $(>) ;\n",
-"case * : UserObject $(<) : $(>) ;\n",
-"}\n",
-"}\n",
-"rule ObjectCcFlags\n",
-"{\n",
-"CCFLAGS on [ FGristFiles $(<:S=$(SUFOBJ)) ] += $(>) ;\n",
-"}\n",
-"rule ObjectC++Flags\n",
-"{\n",
-"C++FLAGS on [ FGristFiles $(<:S=$(SUFOBJ)) ] += $(>) ;\n",
-"}\n",
-"rule ObjectHdrs\n",
-"{\n",
-"HDRS on [ FGristFiles $(<:S=$(SUFOBJ)) ] += $(>) ;\n",
-"}\n",
-"rule Objects\n",
-"{\n",
-"local _i ;\n",
-"for _i in [ FGristFiles $(<) ]\n",
-"{\n",
-"Object $(_i:S=$(SUFOBJ)) : $(_i) ;\n",
-"DEPENDS obj : $(_i:S=$(SUFOBJ)) ;\n",
-"}\n",
-"}\n",
-"rule RmTemps\n",
-"{\n",
-"TEMPORARY $(>) ;\n",
-"}\n",
-"rule Setuid\n",
-"{\n",
-"MODE on [ FAppendSuffix $(<) : $(SUFEXE) ] = 4711 ;\n",
-"}\n",
-"rule Shell\n",
-"{\n",
-"DEPENDS shell : $(<) ;\n",
-"DEPENDS $(<) : $(>) ;\n",
-"SEARCH on $(>) = $(SEARCH_SOURCE) ;\n",
-"MODE on $(<) = $(SHELLMODE) ;\n",
-"Clean clean : $(<) ;\n",
-"Chmod $(<) ;\n",
-"}\n",
-"rule SubDir\n",
-"{\n",
-"local _r _s ;\n",
-"if ! $($(<[1]))\n",
-"{\n",
-"if ! $(<[1])\n",
-"{\n",
-"EXIT SubDir syntax error ;\n",
-"}\n",
-"$(<[1]) = [ FSubDir $(<[2-]) ] ;\n",
-"}\n",
-"if ! $($(<[1])-included)\n",
-"{\n",
-"$(<[1])-included = TRUE ;\n",
-"_r = $($(<[1])RULES) ;\n",
-"if ! $(_r)\n",
-"{\n",
-"_r = $(JAMRULES:R=$($(<[1]))) ;\n",
-"}\n",
-"include $(_r) ;\n",
-"}\n",
-"_s = [ FDirName $(<[2-]) ] ;\n",
-"SUBDIR = $(_s:R=$($(<[1]))) ;\n",
-"SUBDIR_TOKENS = $(<[2-]) ;\n",
-"SEARCH_SOURCE = $(SUBDIR) ;\n",
-"LOCATE_SOURCE = $(ALL_LOCATE_TARGET) $(SUBDIR) ;\n",
-"LOCATE_TARGET = $(ALL_LOCATE_TARGET) $(SUBDIR) ;\n",
-"SOURCE_GRIST = [ FGrist $(<[2-]) ] ;\n",
-"SUBDIRCCFLAGS = ;\n",
-"SUBDIRC++FLAGS = ;\n",
-"SUBDIRHDRS = ;\n",
-"}\n",
-"rule SubDirCcFlags\n",
-"{\n",
-"SUBDIRCCFLAGS += $(<) ;\n",
-"}\n",
-"rule SubDirC++Flags\n",
-"{\n",
-"SUBDIRC++FLAGS += $(<) ;\n",
-"}\n",
-"rule SubDirHdrs\n",
-"{\n",
-"SUBDIRHDRS += $(<) ;\n",
-"}\n",
-"rule SubInclude\n",
-"{\n",
-"local _s ;\n",
-"if ! $($(<[1]))\n",
-"{\n",
-"EXIT Top level of source tree has not been set with $(<[1]) ;\n",
-"}\n",
-"_s = [ FDirName $(<[2-]) ] ;\n",
-"include $(JAMFILE:D=$(_s):R=$($(<[1]))) ;\n",
-"}\n",
-"rule Undefines\n",
-"{\n",
-"UNDEFS on [ FAppendSuffix $(<) : $(SUFEXE) ] += $(UNDEFFLAG)$(>) ;\n",
-"}\n",
-"rule UserObject\n",
-"{\n",
-"EXIT \"Unknown suffix on\" $(>) \"- see UserObject rule in Jamfile(5).\" ;\n",
-"}\n",
-"rule Yacc\n",
-"{\n",
-"local _h ;\n",
-"_h = $(<:BS=.h) ;\n",
-"MakeLocate $(<) $(_h) : $(LOCATE_SOURCE) ;\n",
-"if $(YACC)\n",
-"{\n",
-"DEPENDS $(<) $(_h) : $(>) ;\n",
-"Yacc1 $(<) $(_h) : $(>) ;\n",
-"YaccMv $(<) $(_h) : $(>) ;\n",
-"Clean clean : $(<) $(_h) ;\n",
-"}\n",
-"INCLUDES $(<) : $(_h) ;\n",
-"}\n",
-"rule FGrist\n",
-"{\n",
-"local _g _i ;\n",
-"_g = $(<[1]) ;\n",
-"for _i in $(<[2-])\n",
-"{\n",
-"_g = $(_g)!$(_i) ;\n",
-"}\n",
-"return $(_g) ;\n",
-"}\n",
-"rule FGristFiles\n",
-"{\n",
-"if ! $(SOURCE_GRIST)\n",
-"{\n",
-"return $(<) ;\n",
-"}\n",
-"else\n",
-"{\n",
-"return $(<:G=$(SOURCE_GRIST)) ;\n",
-"}\n",
-"}\n",
-"rule FGristSourceFiles\n",
-"{\n",
-"if ! $(SOURCE_GRIST)\n",
-"{\n",
-"return $(<) ;\n",
-"}\n",
-"else\n",
-"{\n",
-"local _i _o ;\n",
-"for _i in $(<)\n",
-"{\n",
-"switch $(_i)\n",
-"{\n",
-"case *.h : _o += $(_i) ;\n",
-"case * : _o += $(_i:G=$(SOURCE_GRIST)) ;\n",
-"}\n",
-"}\n",
-"return $(_o) ;\n",
-"}\n",
-"}\n",
-"rule FConcat\n",
-"{\n",
-"local _t _r ;\n",
-"$(_r) = $(<[1]) ;\n",
-"for _t in $(<[2-])\n",
-"{\n",
-"$(_r) = $(_r)$(_t) ;\n",
-"}\n",
-"return $(_r) ;\n",
-"}\n",
-"rule FSubDir\n",
-"{\n",
-"local _i _d ;\n",
-"if ! $(<[1])\n",
-"{\n",
-"_d = $(DOT) ;\n",
-"}\n",
-"else\n",
-"{\n",
-"_d = $(DOTDOT) ;\n",
-"for _i in $(<[2-])\n",
-"{\n",
-"_d = $(_d:R=$(DOTDOT)) ;\n",
-"}\n",
-"}\n",
-"return $(_d) ;\n",
-"}\n",
-"rule FDirName\n",
-"{\n",
-"local _s _i ;\n",
-"if ! $(<)\n",
-"{\n",
-"_s = $(DOT) ;\n",
-"}\n",
-"else if $(VMS)\n",
-"{\n",
-"switch $(<[1])\n",
-"{\n",
-"case *:* : _s = $(<[1]) ;\n",
-"case \\\\[*\\\\] : _s = $(<[1]) ;\n",
-"case * : _s = [.$(<[1])] ;\n",
-"}\n",
-"for _i in [.$(<[2-])]\n",
-"{\n",
-"_s = $(_i:R=$(_s)) ;\n",
-"}\n",
-"}\n",
-"else if $(MAC)\n",
-"{\n",
-"_s = $(DOT) ;\n",
-"for _i in $(<)\n",
-"{\n",
-"_s = $(_i:R=$(_s)) ;\n",
-"}\n",
-"}\n",
-"else\n",
-"{\n",
-"_s = $(<[1]) ;\n",
-"for _i in $(<[2-])\n",
-"{\n",
-"_s = $(_i:R=$(_s)) ;\n",
-"}\n",
-"}\n",
-"return $(_s) ;\n",
-"}\n",
-"rule _makeCommon\n",
-"{\n",
-"if $($(<)[1]) && $($(<)[1]) = $($(>)[1])\n",
-"{\n",
-"$(<) = $($(<)[2-]) ;\n",
-"$(>) = $($(>)[2-]) ;\n",
-"_makeCommon $(<) : $(>) ;\n",
-"}\n",
-"}\n",
-"rule FRelPath\n",
-"{\n",
-"local _l _r ;\n",
-"_l = $(<) ;\n",
-"_r = $(>) ;\n",
-"_makeCommon _l : _r ;\n",
-"_l = [ FSubDir $(_l) ] ;\n",
-"_r = [ FDirName $(_r) ] ;\n",
-"if $(_r) = $(DOT) {\n",
-"return $(_l) ;\n",
-"} else {\n",
-"return $(_r:R=$(_l)) ;\n",
-"}\n",
-"}\n",
-"rule FAppendSuffix\n",
-"{\n",
-"if $(>)\n",
-"{\n",
-"local _i _o ;\n",
-"for _i in $(<)\n",
-"{\n",
-"if $(_i:S)\n",
-"{\n",
-"_o += $(_i) ;\n",
-"}\n",
-"else\n",
-"{\n",
-"_o += $(_i:S=$(>)) ;\n",
-"}\n",
-"}\n",
-"return $(_o) ;\n",
-"}\n",
-"else\n",
-"{\n",
-"return $(<) ;\n",
-"}\n",
-"}\n",
-"rule unmakeDir\n",
-"{\n",
-"if $(>[1]:D) && $(>[1]:D) != $(>[1]) && $(>[1]:D) != \\\\\\\\\n",
-"{\n",
-"unmakeDir $(<) : $(>[1]:D) $(>[1]:BS) $(>[2-]) ;\n",
-"}\n",
-"else\n",
-"{\n",
-"$(<) = $(>) ;\n",
-"}\n",
-"}\n",
-"rule FConvertToSlashes\n",
-"{\n",
-"local _d, _s, _i ;\n",
-"unmakeDir _d : $(<) ;\n",
-"_s = $(_d[1]) ;\n",
-"for _i in $(_d[2-])\n",
-"{\n",
-"_s = $(_s)/$(_i) ;\n",
-"}\n",
-"return $(_s) ;\n",
-"}\n",
-"actions updated together piecemeal Archive\n",
-"{\n",
-"$(AR) $(<) $(>)\n",
-"}\n",
-"actions As\n",
-"{\n",
-"$(AS) $(ASFLAGS) -I$(HDRS) -o $(<) $(>)\n",
-"}\n",
-"actions C++\n",
-"{\n",
-"$(C++) -c $(C++FLAGS) $(OPTIM) -I$(HDRS) -o $(<) $(>)\n",
-"}\n",
-"actions Cc\n",
-"{\n",
-"$(CC) -c $(CCFLAGS) $(OPTIM) -I$(HDRS) -o $(<) $(>)\n",
-"}\n",
-"actions Chgrp\n",
-"{\n",
-"$(CHGRP) $(GROUP) $(<)\n",
-"}\n",
-"actions Chmod1\n",
-"{\n",
-"$(CHMOD) $(MODE) $(<)\n",
-"}\n",
-"actions Chown\n",
-"{\n",
-"$(CHOWN) $(OWNER) $(<)\n",
-"}\n",
-"actions piecemeal together existing Clean\n",
-"{\n",
-"$(RM) $(>)\n",
-"}\n",
-"actions File\n",
-"{\n",
-"$(CP) $(>) $(<)\n",
-"}\n",
-"actions GenFile1\n",
-"{\n",
-"$(>[1]) $(<) $(>[2-])\n",
-"}\n",
-"actions Fortran\n",
-"{\n",
-"$(FORTRAN) $(FORTRANFLAGS) -o $(<) $(>)\n",
-"}\n",
-"actions HardLink\n",
-"{\n",
-"$(RM) $(<) && $(LN) $(>) $(<)\n",
-"}\n",
-"actions Install\n",
-"{\n",
-"$(CP) $(>) $(<)\n",
-"}\n",
-"actions Lex\n",
-"{\n",
-"$(LEX) $(>)\n",
-"}\n",
-"actions LexMv\n",
-"{\n",
-"$(MV) lex.yy.c $(<)\n",
-"}\n",
-"actions Link bind NEEDLIBS\n",
-"{\n",
-"$(LINK) $(LINKFLAGS) -o $(<) $(UNDEFS) $(>) $(NEEDLIBS) $(LINKLIBS)\n",
-"}\n",
-"actions MkDir1\n",
-"{\n",
-"$(MKDIR) $(<)\n",
-"}\n",
-"actions together Ranlib\n",
-"{\n",
-"$(RANLIB) $(<)\n",
-"}\n",
-"actions quietly updated piecemeal together RmTemps\n",
-"{\n",
-"$(RM) $(>)\n",
-"}\n",
-"actions Shell\n",
-"{\n",
-"$(AWK) '\n",
-"NR == 1 { print \"$(SHELLHEADER)\" }\n",
-"NR == 1 && /^[#:]/ { next }\n",
-"/^##/ { next }\n",
-"{ print }\n",
-"' < $(>) > $(<)\n",
-"}\n",
-"actions Yacc1\n",
-"{\n",
-"$(YACC) $(YACCFLAGS) $(>)\n",
-"}\n",
-"actions YaccMv\n",
-"{\n",
-"$(MV) $(YACCFILES).c $(<[1])\n",
-"$(MV) $(YACCFILES).h $(<[2])\n",
-"}\n",
-"if $(RELOCATE)\n",
-"{\n",
-"actions C++\n",
-"{\n",
-"$(C++) -c $(C++FLAGS) $(OPTIM) -I$(HDRS) $(>)\n",
-"}\n",
-"actions Cc\n",
-"{\n",
-"$(CC) -c $(CCFLAGS) $(OPTIM) -I$(HDRS) $(>)\n",
-"}\n",
-"actions ignore CcMv\n",
-"{\n",
-"[ $(<) != $(>:BS=$(SUFOBJ)) ] && $(MV) $(>:BS=$(SUFOBJ)) $(<)\n",
-"}\n",
-"}\n",
-"if $(NOARUPDATE)\n",
-"{\n",
-"actions Archive\n",
-"{\n",
-"$(AR) $(<) $(>)\n",
-"}\n",
-"}\n",
-"if $(NT)\n",
-"{\n",
-"if $(TOOLSET) = VISUALC || $(TOOLSET) = VC7 || $(TOOLSET) = INTELC\n",
-"{\n",
-"actions updated together piecemeal Archive\n",
-"{\n",
-"if exist $(<) set _$(<:B)_=$(<)\n",
-"$(AR) /out:$(<) %_$(<:B)_% $(>)\n",
-"}\n",
-"actions As\n",
-"{\n",
-"$(AS) /Ml /p /v /w2 $(>) $(<) ,nul,nul;\n",
-"}\n",
-"actions Cc\n",
-"{\n",
-"$(CC) /c $(CCFLAGS) $(OPTIM) /Fo$(<) /I$(HDRS) /I$(STDHDRS) $(>)\n",
-"}\n",
-"actions C++\n",
-"{\n",
-"$(C++) /c $(C++FLAGS) $(OPTIM) /Fo$(<) /I$(HDRS) /I$(STDHDRS) /Tp$(>)\n",
-"}\n",
-"actions Link bind NEEDLIBS\n",
-"{\n",
-"$(LINK) $(LINKFLAGS) /out:$(<) $(UNDEFS) $(>) $(NEEDLIBS) $(LINKLIBS)\n",
-"}\n",
-"}\n",
-"else if $(TOOLSET) = VISUALC16\n",
-"{\n",
-"actions updated together piecemeal Archive\n",
-"{\n",
-"$(AR) $(<) -+$(>)\n",
-"}\n",
-"actions Cc\n",
-"{\n",
-"$(CC) /c $(CCFLAGS) $(OPTIM) /Fo$(<) /I$(HDRS) $(>)\n",
-"}\n",
-"actions C++\n",
-"{\n",
-"$(C++) /c $(C++FLAGS) $(OPTIM) /Fo$(<) /I$(HDRS) /Tp$(>)\n",
-"}\n",
-"actions Link bind NEEDLIBS\n",
-"{\n",
-"$(LINK) $(LINKFLAGS) /out:$(<) $(UNDEFS) $(>) $(NEEDLIBS) $(LINKLIBS)\n",
-"}\n",
-"}\n",
-"else if $(TOOLSET) = BORLANDC\n",
-"{\n",
-"actions updated together piecemeal Archive\n",
-"{\n",
-"$(AR) $(<) -+$(>)\n",
-"}\n",
-"actions Link bind NEEDLIBS\n",
-"{\n",
-"$(LINK) -e$(<) $(LINKFLAGS) $(UNDEFS) -L$(LINKLIBS) $(NEEDLIBS) $(>)\n",
-"}\n",
-"actions C++\n",
-"{\n",
-"$(C++) -c $(C++FLAGS) $(OPTIM) -I$(HDRS) -o$(<) $(>)\n",
-"}\n",
-"actions Cc\n",
-"{\n",
-"$(CC) -c $(CCFLAGS) $(OPTIM) -I$(HDRS) -o$(<) $(>)\n",
-"}\n",
-"}\n",
-"else if $(TOOLSET) = MINGW\n",
-"{\n",
-"actions together piecemeal Archive\n",
-"{\n",
-"$(AR) $(<) $(>:T)\n",
-"}\n",
-"actions Cc\n",
-"{\n",
-"$(CC) -c $(CCFLAGS) $(OPTIM) -I$(HDRS) -o$(<) $(>)\n",
-"}\n",
-"actions C++\n",
-"{\n",
-"$(C++) -c $(C++FLAGS) $(OPTIM) -I$(HDRS) -o$(<) $(>)\n",
-"}\n",
-"}\n",
-"else if $(TOOLSET) = WATCOM\n",
-"{\n",
-"actions together piecemeal Archive\n",
-"{\n",
-"$(AR) $(<) +-$(>)\n",
-"}\n",
-"actions Cc\n",
-"{\n",
-"$(CC) $(CCFLAGS) $(OPTIM) /Fo=$(<) /I$(HDRS) $(>)\n",
-"}\n",
-"actions C++\n",
-"{\n",
-"$(C++) $(C++FLAGS) $(OPTIM) /Fo=$(<) /I$(HDRS) $(>)\n",
-"}\n",
-"actions Link bind NEEDLIBS\n",
-"{\n",
-"$(LINK) $(LINKFLAGS) /Fe=$(<) $(UNDEFS) $(>) $(NEEDLIBS) $(LINKLIBS)\n",
-"}\n",
-"actions Shell\n",
-"{\n",
-"$(CP) $(>) $(<)\n",
-"}\n",
-"}\n",
-"else if $(TOOLSET) = LCC\n",
-"{\n",
-"actions together piecemeal Archive\n",
-"{\n",
-"$(AR) /out:$(<) $(>)\n",
-"}\n",
-"actions Cc\n",
-"{\n",
-"$(CC) $(CCFLAGS) $(OPTIM) -Fo$(<) -I$(HDRS) $(>)\n",
-"}\n",
-"actions Link bind NEEDLIBS\n",
-"{\n",
-"$(LINK) $(LINKFLAGS) -o $(<) $(UNDEFS) $(>) $(NEEDLIBS) $(LINKLIBS)\n",
-"}\n",
-"actions Shell\n",
-"{\n",
-"$(CP) $(>) $(<)\n",
-"}\n",
-"}\n",
-"}\n",
-"else if $(OS2)\n",
-"{\n",
-"if $(TOOLSET) = WATCOM\n",
-"{\n",
-"actions together piecemeal Archive\n",
-"{\n",
-"$(AR) $(<) +-$(>)\n",
-"}\n",
-"actions Cc\n",
-"{\n",
-"$(CC) $(CCFLAGS) $(OPTIM) /Fo=$(<) /I$(HDRS) $(>)\n",
-"}\n",
-"actions C++\n",
-"{\n",
-"$(C++) $(C++FLAGS) $(OPTIM) /Fo=$(<) /I$(HDRS) $(>)\n",
-"}\n",
-"actions Link bind NEEDLIBS\n",
-"{\n",
-"$(LINK) $(LINKFLAGS) /Fe=$(<) $(UNDEFS) $(>) $(NEEDLIBS) $(LINKLIBS)\n",
-"}\n",
-"actions Shell\n",
-"{\n",
-"$(CP) $(>) $(<)\n",
-"}\n",
-"}\n",
-"else if $(TOOLSET) = EMX\n",
-"{\n",
-"actions together piecemeal Archive\n",
-"{\n",
-"$(AR) $(<) $(>:T)\n",
-"}\n",
-"actions Cc\n",
-"{\n",
-"$(CC) -c $(CCFLAGS) $(OPTIM) -I$(HDRS) -o$(<) $(>)\n",
-"}\n",
-"actions C++\n",
-"{\n",
-"$(C++) -c $(C++FLAGS) $(OPTIM) -I$(HDRS) -o$(<) $(>)\n",
-"}\n",
-"}\n",
-"}\n",
-"else if $(VMS)\n",
-"{\n",
-"actions updated together piecemeal Archive\n",
-"{\n",
-"lib/replace $(<) $(>[1]) ,$(>[2-])\n",
-"}\n",
-"actions Cc\n",
-"{\n",
-"$(CC)/obj=$(<) $(CCFLAGS) $(OPTIM) $(SLASHINC) $(>)\n",
-"}\n",
-"actions C++\n",
-"{\n",
-"$(C++)/obj=$(<) $(C++FLAGS) $(OPTIM) $(SLASHINC) $(>)\n",
-"}\n",
-"actions piecemeal together existing Clean\n",
-"{\n",
-"$(RM) $(>[1]);* ,$(>[2-]);*\n",
-"}\n",
-"actions together quietly CreLib\n",
-"{\n",
-"if f$search(\"$(<)\") .eqs. \"\" then lib/create $(<)\n",
-"}\n",
-"actions GenFile1\n",
-"{\n",
-"mcr $(>[1]) $(<) $(>[2-])\n",
-"}\n",
-"actions Link bind NEEDLIBS\n",
-"{\n",
-"$(LINK)/exe=$(<) $(LINKFLAGS) $(>[1]) ,$(>[2-]) ,$(NEEDLIBS)/lib ,$(LINKLIBS)\n",
-"}\n",
-"actions quietly updated piecemeal together RmTemps\n",
-"{\n",
-"$(RM) $(>[1]);* ,$(>[2-]);*\n",
-"}\n",
-"actions Shell\n",
-"{\n",
-"$(CP) $(>) $(<)\n",
-"}\n",
-"}\n",
-"else if $(MAC)\n",
-"{\n",
-"actions together Archive\n",
-"{\n",
-"$(LINK) -library -o $(<) $(>)\n",
-"}\n",
-"actions Cc\n",
-"{\n",
-"set -e MWCincludes $(MACINC)\n",
-"$(CC) -o $(<) $(CCFLAGS) $(OPTIM) $(>)\n",
-"}\n",
-"actions C++\n",
-"{\n",
-"set -e MWCincludes $(MACINC)\n",
-"$(CC) -o $(<) $(C++FLAGS) $(OPTIM) $(>)\n",
-"}\n",
-"actions Link bind NEEDLIBS\n",
-"{\n",
-"$(LINK) -o $(<) $(LINKFLAGS) $(>) $(NEEDLIBS) \"$(LINKLIBS)\"\n",
-"}\n",
-"}\n",
-"rule BULK { Bulk $(<) : $(>) ; }\n",
-"rule FILE { File $(<) : $(>) ; }\n",
-"rule HDRRULE { HdrRule $(<) : $(>) ; }\n",
-"rule INSTALL { Install $(<) : $(>) ; }\n",
-"rule LIBRARY { Library $(<) : $(>) ; }\n",
-"rule LIBS { LinkLibraries $(<) : $(>) ; }\n",
-"rule LINK { Link $(<) : $(>) ; }\n",
-"rule MAIN { Main $(<) : $(>) ; }\n",
-"rule SETUID { Setuid $(<) ; }\n",
-"rule SHELL { Shell $(<) : $(>) ; }\n",
-"rule UNDEFINES { Undefines $(<) : $(>) ; }\n",
-"rule INSTALLBIN { InstallBin $(BINDIR) : $(<) ; }\n",
-"rule INSTALLLIB { InstallLib $(LIBDIR) : $(<) ; }\n",
-"rule INSTALLMAN { InstallMan $(MANDIR) : $(<) ; }\n",
-"rule addDirName { $(<) += [ FDirName $(>) ] ; }\n",
-"rule makeDirName { $(<) = [ FDirName $(>) ] ; }\n",
-"rule makeGristedName { $(<) = [ FGristSourceFiles $(>) ] ; }\n",
-"rule makeRelPath { $(<[1]) = [ FRelPath $(<[2-]) : $(>) ] ; }\n",
-"rule makeSuffixed { $(<[1]) = [ FAppendSuffix $(>) : $(<[2]) ] ; }\n",
-"{\n",
-"if $(JAMFILE) { include $(JAMFILE) ; }\n",
-"}\n",
-"}\n",
-0 };
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/jambase.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/jambase.h b/ext/kenlm/jam-files/engine/jambase.h
deleted file mode 100644
index c05ec79..0000000
--- a/ext/kenlm/jam-files/engine/jambase.h
+++ /dev/null
@@ -1,15 +0,0 @@
-/*
- * Copyright 1993, 1995 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/*
- * jambase.h - declaration for the internal jambase
- *
- * The file Jambase is turned into a C array of strings in jambase.c
- * so that it can be built in to the executable. This is the
- * declaration for that array.
- */
-
-extern char *jambase[];
[06/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/python/kenlm.cpp
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/python/kenlm.cpp b/ext/kenlm/python/kenlm.cpp
deleted file mode 100644
index 5bf47ff..0000000
--- a/ext/kenlm/python/kenlm.cpp
+++ /dev/null
@@ -1,7379 +0,0 @@
-/* Generated by Cython 0.22 */
-
-#define PY_SSIZE_T_CLEAN
-#ifndef CYTHON_USE_PYLONG_INTERNALS
-#ifdef PYLONG_BITS_IN_DIGIT
-#define CYTHON_USE_PYLONG_INTERNALS 0
-#else
-#include "pyconfig.h"
-#ifdef PYLONG_BITS_IN_DIGIT
-#define CYTHON_USE_PYLONG_INTERNALS 1
-#else
-#define CYTHON_USE_PYLONG_INTERNALS 0
-#endif
-#endif
-#endif
-#include "Python.h"
-#ifndef Py_PYTHON_H
- #error Python headers needed to compile C extensions, please install development version of Python.
-#elif PY_VERSION_HEX < 0x02060000 || (0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03020000)
- #error Cython requires Python 2.6+ or Python 3.2+.
-#else
-#define CYTHON_ABI "0_22"
-#include <stddef.h>
-#ifndef offsetof
-#define offsetof(type, member) ( (size_t) & ((type*)0) -> member )
-#endif
-#if !defined(WIN32) && !defined(MS_WINDOWS)
- #ifndef __stdcall
- #define __stdcall
- #endif
- #ifndef __cdecl
- #define __cdecl
- #endif
- #ifndef __fastcall
- #define __fastcall
- #endif
-#endif
-#ifndef DL_IMPORT
- #define DL_IMPORT(t) t
-#endif
-#ifndef DL_EXPORT
- #define DL_EXPORT(t) t
-#endif
-#ifndef PY_LONG_LONG
- #define PY_LONG_LONG LONG_LONG
-#endif
-#ifndef Py_HUGE_VAL
- #define Py_HUGE_VAL HUGE_VAL
-#endif
-#ifdef PYPY_VERSION
-#define CYTHON_COMPILING_IN_PYPY 1
-#define CYTHON_COMPILING_IN_CPYTHON 0
-#else
-#define CYTHON_COMPILING_IN_PYPY 0
-#define CYTHON_COMPILING_IN_CPYTHON 1
-#endif
-#if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x02070600 && !defined(Py_OptimizeFlag)
-#define Py_OptimizeFlag 0
-#endif
-#define __PYX_BUILD_PY_SSIZE_T "n"
-#define CYTHON_FORMAT_SSIZE_T "z"
-#if PY_MAJOR_VERSION < 3
- #define __Pyx_BUILTIN_MODULE_NAME "__builtin__"
- #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \
- PyCode_New(a+k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)
- #define __Pyx_DefaultClassType PyClass_Type
-#else
- #define __Pyx_BUILTIN_MODULE_NAME "builtins"
- #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \
- PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)
- #define __Pyx_DefaultClassType PyType_Type
-#endif
-#if PY_MAJOR_VERSION >= 3
- #define Py_TPFLAGS_CHECKTYPES 0
- #define Py_TPFLAGS_HAVE_INDEX 0
- #define Py_TPFLAGS_HAVE_NEWBUFFER 0
-#endif
-#if PY_VERSION_HEX < 0x030400a1 && !defined(Py_TPFLAGS_HAVE_FINALIZE)
- #define Py_TPFLAGS_HAVE_FINALIZE 0
-#endif
-#if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND)
- #define CYTHON_PEP393_ENABLED 1
- #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ? \
- 0 : _PyUnicode_Ready((PyObject *)(op)))
- #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_LENGTH(u)
- #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i)
- #define __Pyx_PyUnicode_KIND(u) PyUnicode_KIND(u)
- #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u)
- #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i)
-#else
- #define CYTHON_PEP393_ENABLED 0
- #define __Pyx_PyUnicode_READY(op) (0)
- #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_SIZE(u)
- #define __Pyx_PyUnicode_READ_CHAR(u, i) ((Py_UCS4)(PyUnicode_AS_UNICODE(u)[i]))
- #define __Pyx_PyUnicode_KIND(u) (sizeof(Py_UNICODE))
- #define __Pyx_PyUnicode_DATA(u) ((void*)PyUnicode_AS_UNICODE(u))
- #define __Pyx_PyUnicode_READ(k, d, i) ((void)(k), (Py_UCS4)(((Py_UNICODE*)d)[i]))
-#endif
-#if CYTHON_COMPILING_IN_PYPY
- #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b)
- #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b)
- #define __Pyx_PyFrozenSet_Size(s) PyObject_Size(s)
-#else
- #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b)
- #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ? \
- PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b))
- #define __Pyx_PyFrozenSet_Size(s) PySet_Size(s)
-#endif
-#define __Pyx_PyString_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : __Pyx_PyString_Format(a, b))
-#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b))
-#if PY_MAJOR_VERSION >= 3
- #define __Pyx_PyString_Format(a, b) PyUnicode_Format(a, b)
-#else
- #define __Pyx_PyString_Format(a, b) PyString_Format(a, b)
-#endif
-#if PY_MAJOR_VERSION >= 3
- #define PyBaseString_Type PyUnicode_Type
- #define PyStringObject PyUnicodeObject
- #define PyString_Type PyUnicode_Type
- #define PyString_Check PyUnicode_Check
- #define PyString_CheckExact PyUnicode_CheckExact
-#endif
-#if PY_MAJOR_VERSION >= 3
- #define __Pyx_PyBaseString_Check(obj) PyUnicode_Check(obj)
- #define __Pyx_PyBaseString_CheckExact(obj) PyUnicode_CheckExact(obj)
-#else
- #define __Pyx_PyBaseString_Check(obj) (PyString_Check(obj) || PyUnicode_Check(obj))
- #define __Pyx_PyBaseString_CheckExact(obj) (PyString_CheckExact(obj) || PyUnicode_CheckExact(obj))
-#endif
-#ifndef PySet_CheckExact
- #define PySet_CheckExact(obj) (Py_TYPE(obj) == &PySet_Type)
-#endif
-#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type)
-#if PY_MAJOR_VERSION >= 3
- #define PyIntObject PyLongObject
- #define PyInt_Type PyLong_Type
- #define PyInt_Check(op) PyLong_Check(op)
- #define PyInt_CheckExact(op) PyLong_CheckExact(op)
- #define PyInt_FromString PyLong_FromString
- #define PyInt_FromUnicode PyLong_FromUnicode
- #define PyInt_FromLong PyLong_FromLong
- #define PyInt_FromSize_t PyLong_FromSize_t
- #define PyInt_FromSsize_t PyLong_FromSsize_t
- #define PyInt_AsLong PyLong_AsLong
- #define PyInt_AS_LONG PyLong_AS_LONG
- #define PyInt_AsSsize_t PyLong_AsSsize_t
- #define PyInt_AsUnsignedLongMask PyLong_AsUnsignedLongMask
- #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask
- #define PyNumber_Int PyNumber_Long
-#endif
-#if PY_MAJOR_VERSION >= 3
- #define PyBoolObject PyLongObject
-#endif
-#if PY_MAJOR_VERSION >= 3 && CYTHON_COMPILING_IN_PYPY
- #ifndef PyUnicode_InternFromString
- #define PyUnicode_InternFromString(s) PyUnicode_FromString(s)
- #endif
-#endif
-#if PY_VERSION_HEX < 0x030200A4
- typedef long Py_hash_t;
- #define __Pyx_PyInt_FromHash_t PyInt_FromLong
- #define __Pyx_PyInt_AsHash_t PyInt_AsLong
-#else
- #define __Pyx_PyInt_FromHash_t PyInt_FromSsize_t
- #define __Pyx_PyInt_AsHash_t PyInt_AsSsize_t
-#endif
-#if PY_MAJOR_VERSION >= 3
- #define __Pyx_PyMethod_New(func, self, klass) ((self) ? PyMethod_New(func, self) : PyInstanceMethod_New(func))
-#else
- #define __Pyx_PyMethod_New(func, self, klass) PyMethod_New(func, self, klass)
-#endif
-#ifndef CYTHON_INLINE
- #if defined(__GNUC__)
- #define CYTHON_INLINE __inline__
- #elif defined(_MSC_VER)
- #define CYTHON_INLINE __inline
- #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
- #define CYTHON_INLINE inline
- #else
- #define CYTHON_INLINE
- #endif
-#endif
-#ifndef CYTHON_RESTRICT
- #if defined(__GNUC__)
- #define CYTHON_RESTRICT __restrict__
- #elif defined(_MSC_VER) && _MSC_VER >= 1400
- #define CYTHON_RESTRICT __restrict
- #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
- #define CYTHON_RESTRICT restrict
- #else
- #define CYTHON_RESTRICT
- #endif
-#endif
-#ifdef NAN
-#define __PYX_NAN() ((float) NAN)
-#else
-static CYTHON_INLINE float __PYX_NAN() {
- /* Initialize NaN. The sign is irrelevant, an exponent with all bits 1 and
- a nonzero mantissa means NaN. If the first bit in the mantissa is 1, it is
- a quiet NaN. */
- float value;
- memset(&value, 0xFF, sizeof(value));
- return value;
-}
-#endif
-#define __Pyx_void_to_None(void_result) (void_result, Py_INCREF(Py_None), Py_None)
-#ifdef __cplusplus
-template<typename T>
-void __Pyx_call_destructor(T* x) {
- x->~T();
-}
-template<typename T>
-class __Pyx_FakeReference {
- public:
- __Pyx_FakeReference() : ptr(NULL) { }
- __Pyx_FakeReference(T& ref) : ptr(&ref) { }
- T *operator->() { return ptr; }
- operator T&() { return *ptr; }
- private:
- T *ptr;
-};
-#endif
-
-
-#if PY_MAJOR_VERSION >= 3
- #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y)
- #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y)
-#else
- #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y)
- #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceDivide(x,y)
-#endif
-
-#ifndef __PYX_EXTERN_C
- #ifdef __cplusplus
- #define __PYX_EXTERN_C extern "C"
- #else
- #define __PYX_EXTERN_C extern
- #endif
-#endif
-
-#if defined(WIN32) || defined(MS_WINDOWS)
-#define _USE_MATH_DEFINES
-#endif
-#include <math.h>
-#define __PYX_HAVE__kenlm
-#define __PYX_HAVE_API__kenlm
-#include "lm/word_index.hh"
-#include "lm/return.hh"
-#include "lm/state.hh"
-#include "lm/virtual_interface.hh"
-#include "lm/model.hh"
-#include "ios"
-#include "new"
-#include "stdexcept"
-#include "typeinfo"
-#ifdef _OPENMP
-#include <omp.h>
-#endif /* _OPENMP */
-
-#ifdef PYREX_WITHOUT_ASSERTIONS
-#define CYTHON_WITHOUT_ASSERTIONS
-#endif
-
-#ifndef CYTHON_UNUSED
-# if defined(__GNUC__)
-# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4))
-# define CYTHON_UNUSED __attribute__ ((__unused__))
-# else
-# define CYTHON_UNUSED
-# endif
-# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER))
-# define CYTHON_UNUSED __attribute__ ((__unused__))
-# else
-# define CYTHON_UNUSED
-# endif
-#endif
-typedef struct {PyObject **p; char *s; const Py_ssize_t n; const char* encoding;
- const char is_unicode; const char is_str; const char intern; } __Pyx_StringTabEntry;
-
-#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0
-#define __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT 0
-#define __PYX_DEFAULT_STRING_ENCODING ""
-#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString
-#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize
-#define __Pyx_fits_Py_ssize_t(v, type, is_signed) ( \
- (sizeof(type) < sizeof(Py_ssize_t)) || \
- (sizeof(type) > sizeof(Py_ssize_t) && \
- likely(v < (type)PY_SSIZE_T_MAX || \
- v == (type)PY_SSIZE_T_MAX) && \
- (!is_signed || likely(v > (type)PY_SSIZE_T_MIN || \
- v == (type)PY_SSIZE_T_MIN))) || \
- (sizeof(type) == sizeof(Py_ssize_t) && \
- (is_signed || likely(v < (type)PY_SSIZE_T_MAX || \
- v == (type)PY_SSIZE_T_MAX))) )
-static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject*);
-static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length);
-#define __Pyx_PyByteArray_FromString(s) PyByteArray_FromStringAndSize((const char*)s, strlen((const char*)s))
-#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l)
-#define __Pyx_PyBytes_FromString PyBytes_FromString
-#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize
-static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*);
-#if PY_MAJOR_VERSION < 3
- #define __Pyx_PyStr_FromString __Pyx_PyBytes_FromString
- #define __Pyx_PyStr_FromStringAndSize __Pyx_PyBytes_FromStringAndSize
-#else
- #define __Pyx_PyStr_FromString __Pyx_PyUnicode_FromString
- #define __Pyx_PyStr_FromStringAndSize __Pyx_PyUnicode_FromStringAndSize
-#endif
-#define __Pyx_PyObject_AsSString(s) ((signed char*) __Pyx_PyObject_AsString(s))
-#define __Pyx_PyObject_AsUString(s) ((unsigned char*) __Pyx_PyObject_AsString(s))
-#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s)
-#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s)
-#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s)
-#define __Pyx_PyStr_FromCString(s) __Pyx_PyStr_FromString((const char*)s)
-#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s)
-#if PY_MAJOR_VERSION < 3
-static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u)
-{
- const Py_UNICODE *u_end = u;
- while (*u_end++) ;
- return (size_t)(u_end - u - 1);
-}
-#else
-#define __Pyx_Py_UNICODE_strlen Py_UNICODE_strlen
-#endif
-#define __Pyx_PyUnicode_FromUnicode(u) PyUnicode_FromUnicode(u, __Pyx_Py_UNICODE_strlen(u))
-#define __Pyx_PyUnicode_FromUnicodeAndLength PyUnicode_FromUnicode
-#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode
-#define __Pyx_Owned_Py_None(b) (Py_INCREF(Py_None), Py_None)
-#define __Pyx_PyBool_FromLong(b) ((b) ? (Py_INCREF(Py_True), Py_True) : (Py_INCREF(Py_False), Py_False))
-static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*);
-static CYTHON_INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x);
-static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*);
-static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t);
-#if CYTHON_COMPILING_IN_CPYTHON
-#define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x))
-#else
-#define __pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x)
-#endif
-#define __pyx_PyFloat_AsFloat(x) ((float) __pyx_PyFloat_AsDouble(x))
-#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII
-static int __Pyx_sys_getdefaultencoding_not_ascii;
-static int __Pyx_init_sys_getdefaultencoding_params(void) {
- PyObject* sys;
- PyObject* default_encoding = NULL;
- PyObject* ascii_chars_u = NULL;
- PyObject* ascii_chars_b = NULL;
- const char* default_encoding_c;
- sys = PyImport_ImportModule("sys");
- if (!sys) goto bad;
- default_encoding = PyObject_CallMethod(sys, (char*) "getdefaultencoding", NULL);
- Py_DECREF(sys);
- if (!default_encoding) goto bad;
- default_encoding_c = PyBytes_AsString(default_encoding);
- if (!default_encoding_c) goto bad;
- if (strcmp(default_encoding_c, "ascii") == 0) {
- __Pyx_sys_getdefaultencoding_not_ascii = 0;
- } else {
- char ascii_chars[128];
- int c;
- for (c = 0; c < 128; c++) {
- ascii_chars[c] = c;
- }
- __Pyx_sys_getdefaultencoding_not_ascii = 1;
- ascii_chars_u = PyUnicode_DecodeASCII(ascii_chars, 128, NULL);
- if (!ascii_chars_u) goto bad;
- ascii_chars_b = PyUnicode_AsEncodedString(ascii_chars_u, default_encoding_c, NULL);
- if (!ascii_chars_b || !PyBytes_Check(ascii_chars_b) || memcmp(ascii_chars, PyBytes_AS_STRING(ascii_chars_b), 128) != 0) {
- PyErr_Format(
- PyExc_ValueError,
- "This module compiled with c_string_encoding=ascii, but default encoding '%.200s' is not a superset of ascii.",
- default_encoding_c);
- goto bad;
- }
- Py_DECREF(ascii_chars_u);
- Py_DECREF(ascii_chars_b);
- }
- Py_DECREF(default_encoding);
- return 0;
-bad:
- Py_XDECREF(default_encoding);
- Py_XDECREF(ascii_chars_u);
- Py_XDECREF(ascii_chars_b);
- return -1;
-}
-#endif
-#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT && PY_MAJOR_VERSION >= 3
-#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL)
-#else
-#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL)
-#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT
-static char* __PYX_DEFAULT_STRING_ENCODING;
-static int __Pyx_init_sys_getdefaultencoding_params(void) {
- PyObject* sys;
- PyObject* default_encoding = NULL;
- char* default_encoding_c;
- sys = PyImport_ImportModule("sys");
- if (!sys) goto bad;
- default_encoding = PyObject_CallMethod(sys, (char*) (const char*) "getdefaultencoding", NULL);
- Py_DECREF(sys);
- if (!default_encoding) goto bad;
- default_encoding_c = PyBytes_AsString(default_encoding);
- if (!default_encoding_c) goto bad;
- __PYX_DEFAULT_STRING_ENCODING = (char*) malloc(strlen(default_encoding_c));
- if (!__PYX_DEFAULT_STRING_ENCODING) goto bad;
- strcpy(__PYX_DEFAULT_STRING_ENCODING, default_encoding_c);
- Py_DECREF(default_encoding);
- return 0;
-bad:
- Py_XDECREF(default_encoding);
- return -1;
-}
-#endif
-#endif
-
-
-/* Test for GCC > 2.95 */
-#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95)))
- #define likely(x) __builtin_expect(!!(x), 1)
- #define unlikely(x) __builtin_expect(!!(x), 0)
-#else /* !__GNUC__ or GCC < 2.95 */
- #define likely(x) (x)
- #define unlikely(x) (x)
-#endif /* __GNUC__ */
-
-static PyObject *__pyx_m;
-static PyObject *__pyx_d;
-static PyObject *__pyx_b;
-static PyObject *__pyx_empty_tuple;
-static PyObject *__pyx_empty_bytes;
-static int __pyx_lineno;
-static int __pyx_clineno = 0;
-static const char * __pyx_cfilenm= __FILE__;
-static const char *__pyx_filename;
-
-
-static const char *__pyx_f[] = {
- "kenlm.pyx",
-};
-
-/*--- Type declarations ---*/
-struct __pyx_obj_5kenlm_FullScoreReturn;
-struct __pyx_obj_5kenlm_State;
-struct __pyx_obj_5kenlm_LanguageModel;
-struct __pyx_obj_5kenlm_Model;
-struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores;
-
-/* "kenlm.pyx":11
- * raise TypeError('Cannot convert %s to string' % type(data))
- *
- * cdef class FullScoreReturn: # <<<<<<<<<<<<<<
- * """
- * Wrapper around FullScoreReturn.
- */
-struct __pyx_obj_5kenlm_FullScoreReturn {
- PyObject_HEAD
- float log_prob;
- int ngram_length;
- int oov;
-};
-
-
-/* "kenlm.pyx":44
- * return self.oov
- *
- * cdef class State: # <<<<<<<<<<<<<<
- * """
- * Wrapper around lm::ngram::State so that python code can make incremental queries.
- */
-struct __pyx_obj_5kenlm_State {
- PyObject_HEAD
- lm::ngram::State _c_state;
-};
-
-
-/* "kenlm.pyx":74
- *
- *
- * cdef class LanguageModel: # <<<<<<<<<<<<<<
- * """
- * This is not a strict wrapper, the interface is more pythonic.
- */
-struct __pyx_obj_5kenlm_LanguageModel {
- PyObject_HEAD
- lm::base::Model *model;
- PyObject *path;
- const lm::base::Vocabulary *vocab;
-};
-
-
-/* "kenlm.pyx":159
- * return (_kenlm.LanguageModel, (self.path,))
- *
- * cdef class Model: # <<<<<<<<<<<<<<
- * """
- * This is closer to a wrapper around lm::ngram::Model.
- */
-struct __pyx_obj_5kenlm_Model {
- PyObject_HEAD
- lm::base::Model *model;
- PyObject *path;
- const lm::base::Vocabulary *vocab;
-};
-
-
-/* "kenlm.pyx":122
- * return total
- *
- * def full_scores(self, sentence, bos = True, eos = True): # <<<<<<<<<<<<<<
- * """
- * full_scores(sentence, bos = True, eos = Ture) -> generate full scores (prob, ngram length, oov)
- */
-struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores {
- PyObject_HEAD
- PyObject *__pyx_v_bos;
- PyObject *__pyx_v_eos;
- lm::ngram::State __pyx_v_out_state;
- struct lm::FullScoreReturn __pyx_v_ret;
- struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self;
- PyObject *__pyx_v_sentence;
- lm::ngram::State __pyx_v_state;
- float __pyx_v_total;
- lm::WordIndex __pyx_v_wid;
- PyObject *__pyx_v_word;
- PyObject *__pyx_v_words;
- PyObject *__pyx_t_0;
- Py_ssize_t __pyx_t_1;
-};
-
-
-/* --- Runtime support code (head) --- */
-#ifndef CYTHON_REFNANNY
- #define CYTHON_REFNANNY 0
-#endif
-#if CYTHON_REFNANNY
- typedef struct {
- void (*INCREF)(void*, PyObject*, int);
- void (*DECREF)(void*, PyObject*, int);
- void (*GOTREF)(void*, PyObject*, int);
- void (*GIVEREF)(void*, PyObject*, int);
- void* (*SetupContext)(const char*, int, const char*);
- void (*FinishContext)(void**);
- } __Pyx_RefNannyAPIStruct;
- static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL;
- static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname);
- #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL;
-#ifdef WITH_THREAD
- #define __Pyx_RefNannySetupContext(name, acquire_gil) \
- if (acquire_gil) { \
- PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure(); \
- __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__); \
- PyGILState_Release(__pyx_gilstate_save); \
- } else { \
- __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__); \
- }
-#else
- #define __Pyx_RefNannySetupContext(name, acquire_gil) \
- __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__)
-#endif
- #define __Pyx_RefNannyFinishContext() \
- __Pyx_RefNanny->FinishContext(&__pyx_refnanny)
- #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), __LINE__)
- #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), __LINE__)
- #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), __LINE__)
- #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), __LINE__)
- #define __Pyx_XINCREF(r) do { if((r) != NULL) {__Pyx_INCREF(r); }} while(0)
- #define __Pyx_XDECREF(r) do { if((r) != NULL) {__Pyx_DECREF(r); }} while(0)
- #define __Pyx_XGOTREF(r) do { if((r) != NULL) {__Pyx_GOTREF(r); }} while(0)
- #define __Pyx_XGIVEREF(r) do { if((r) != NULL) {__Pyx_GIVEREF(r);}} while(0)
-#else
- #define __Pyx_RefNannyDeclarations
- #define __Pyx_RefNannySetupContext(name, acquire_gil)
- #define __Pyx_RefNannyFinishContext()
- #define __Pyx_INCREF(r) Py_INCREF(r)
- #define __Pyx_DECREF(r) Py_DECREF(r)
- #define __Pyx_GOTREF(r)
- #define __Pyx_GIVEREF(r)
- #define __Pyx_XINCREF(r) Py_XINCREF(r)
- #define __Pyx_XDECREF(r) Py_XDECREF(r)
- #define __Pyx_XGOTREF(r)
- #define __Pyx_XGIVEREF(r)
-#endif
-#define __Pyx_XDECREF_SET(r, v) do { \
- PyObject *tmp = (PyObject *) r; \
- r = v; __Pyx_XDECREF(tmp); \
- } while (0)
-#define __Pyx_DECREF_SET(r, v) do { \
- PyObject *tmp = (PyObject *) r; \
- r = v; __Pyx_DECREF(tmp); \
- } while (0)
-#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0)
-#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0)
-
-#if CYTHON_COMPILING_IN_CPYTHON
-static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) {
- PyTypeObject* tp = Py_TYPE(obj);
- if (likely(tp->tp_getattro))
- return tp->tp_getattro(obj, attr_name);
-#if PY_MAJOR_VERSION < 3
- if (likely(tp->tp_getattr))
- return tp->tp_getattr(obj, PyString_AS_STRING(attr_name));
-#endif
- return PyObject_GetAttr(obj, attr_name);
-}
-#else
-#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n)
-#endif
-
-static PyObject *__Pyx_GetBuiltinName(PyObject *name);
-
-#if CYTHON_COMPILING_IN_CPYTHON
-static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw);
-#else
-#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw)
-#endif
-
-static CYTHON_INLINE void __Pyx_ErrRestore(PyObject *type, PyObject *value, PyObject *tb);
-static CYTHON_INLINE void __Pyx_ErrFetch(PyObject **type, PyObject **value, PyObject **tb);
-
-static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause);
-
-static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact,
- Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found);
-
-static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name);
-
-static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[], \
- PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args, \
- const char* function_name);
-
-static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed,
- const char *name, int exact);
-
-static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name);
-
-#if CYTHON_COMPILING_IN_CPYTHON
-static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg);
-#endif
-
-static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg);
-
-static CYTHON_INLINE void __Pyx_ExceptionSave(PyObject **type, PyObject **value, PyObject **tb);
-static void __Pyx_ExceptionReset(PyObject *type, PyObject *value, PyObject *tb);
-
-static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb);
-
-#if CYTHON_COMPILING_IN_CPYTHON
-static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func);
-#else
-#define __Pyx_PyObject_CallNoArg(func) __Pyx_PyObject_Call(func, __pyx_empty_tuple, NULL)
-#endif
-
-#include <string.h>
-
-typedef struct {
- int code_line;
- PyCodeObject* code_object;
-} __Pyx_CodeObjectCacheEntry;
-struct __Pyx_CodeObjectCache {
- int count;
- int max_count;
- __Pyx_CodeObjectCacheEntry* entries;
-};
-static struct __Pyx_CodeObjectCache __pyx_code_cache = {0,0,NULL};
-static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line);
-static PyCodeObject *__pyx_find_code_object(int code_line);
-static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object);
-
-static void __Pyx_AddTraceback(const char *funcname, int c_line,
- int py_line, const char *filename);
-
-#include <new>
-
-static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level);
-
-static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *);
-
-static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value);
-
-#ifndef __Pyx_CppExn2PyErr
-#include <new>
-#include <typeinfo>
-#include <stdexcept>
-#include <ios>
-static void __Pyx_CppExn2PyErr() {
- try {
- if (PyErr_Occurred())
- ; // let the latest Python exn pass through and ignore the current one
- else
- throw;
- } catch (const std::bad_alloc& exn) {
- PyErr_SetString(PyExc_MemoryError, exn.what());
- } catch (const std::bad_cast& exn) {
- PyErr_SetString(PyExc_TypeError, exn.what());
- } catch (const std::domain_error& exn) {
- PyErr_SetString(PyExc_ValueError, exn.what());
- } catch (const std::invalid_argument& exn) {
- PyErr_SetString(PyExc_ValueError, exn.what());
- } catch (const std::ios_base::failure& exn) {
- PyErr_SetString(PyExc_IOError, exn.what());
- } catch (const std::out_of_range& exn) {
- PyErr_SetString(PyExc_IndexError, exn.what());
- } catch (const std::overflow_error& exn) {
- PyErr_SetString(PyExc_OverflowError, exn.what());
- } catch (const std::range_error& exn) {
- PyErr_SetString(PyExc_ArithmeticError, exn.what());
- } catch (const std::underflow_error& exn) {
- PyErr_SetString(PyExc_ArithmeticError, exn.what());
- } catch (const std::exception& exn) {
- PyErr_SetString(PyExc_RuntimeError, exn.what());
- }
- catch (...)
- {
- PyErr_SetString(PyExc_RuntimeError, "Unknown exception");
- }
-}
-#endif
-
-static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_int(unsigned int value);
-
-static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_char(unsigned char value);
-
-static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value);
-
-static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *);
-
-static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type);
-
-static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb);
-
-static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg);
-
-#define __Pyx_Generator_USED
-#include <structmember.h>
-#include <frameobject.h>
-typedef PyObject *(*__pyx_generator_body_t)(PyObject *, PyObject *);
-typedef struct {
- PyObject_HEAD
- __pyx_generator_body_t body;
- PyObject *closure;
- PyObject *exc_type;
- PyObject *exc_value;
- PyObject *exc_traceback;
- PyObject *gi_weakreflist;
- PyObject *classobj;
- PyObject *yieldfrom;
- PyObject *gi_name;
- PyObject *gi_qualname;
- int resume_label;
- char is_running;
-} __pyx_GeneratorObject;
-static __pyx_GeneratorObject *__Pyx_Generator_New(__pyx_generator_body_t body,
- PyObject *closure, PyObject *name, PyObject *qualname);
-static int __pyx_Generator_init(void);
-static int __Pyx_Generator_clear(PyObject* self);
-#if 1 || PY_VERSION_HEX < 0x030300B0
-static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue);
-#else
-#define __Pyx_PyGen_FetchStopIterationValue(pvalue) PyGen_FetchStopIterationValue(pvalue)
-#endif
-
-static int __Pyx_check_binary_version(void);
-
-static int __Pyx_InitStrings(__Pyx_StringTabEntry *t);
-
-
-/* Module declarations from '_kenlm' */
-
-/* Module declarations from 'kenlm' */
-static PyTypeObject *__pyx_ptype_5kenlm_FullScoreReturn = 0;
-static PyTypeObject *__pyx_ptype_5kenlm_State = 0;
-static PyTypeObject *__pyx_ptype_5kenlm_LanguageModel = 0;
-static PyTypeObject *__pyx_ptype_5kenlm_Model = 0;
-static PyTypeObject *__pyx_ptype_5kenlm___pyx_scope_struct__full_scores = 0;
-static PyObject *__pyx_f_5kenlm_as_str(PyObject *); /*proto*/
-#define __Pyx_MODULE_NAME "kenlm"
-int __pyx_module_is_main_kenlm = 0;
-
-/* Implementation of 'kenlm' */
-static PyObject *__pyx_builtin_TypeError;
-static PyObject *__pyx_builtin_RuntimeError;
-static PyObject *__pyx_builtin_IOError;
-static int __pyx_pf_5kenlm_15FullScoreReturn___cinit__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self, PyObject *__pyx_v_log_prob, PyObject *__pyx_v_ngram_length, PyObject *__pyx_v_oov); /* proto */
-static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_2__repr__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self); /* proto */
-static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_8log_prob___get__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self); /* proto */
-static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_12ngram_length___get__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self); /* proto */
-static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_3oov___get__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self); /* proto */
-static PyObject *__pyx_pf_5kenlm_5State___richcmp__(struct __pyx_obj_5kenlm_State *__pyx_v_qa, struct __pyx_obj_5kenlm_State *__pyx_v_qb, int __pyx_v_op); /* proto */
-static Py_hash_t __pyx_pf_5kenlm_5State_2__hash__(struct __pyx_obj_5kenlm_State *__pyx_v_self); /* proto */
-static int __pyx_pf_5kenlm_13LanguageModel___init__(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self, PyObject *__pyx_v_path); /* proto */
-static void __pyx_pf_5kenlm_13LanguageModel_2__dealloc__(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self); /* proto */
-static PyObject *__pyx_pf_5kenlm_13LanguageModel_5order___get__(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self); /* proto */
-static PyObject *__pyx_pf_5kenlm_13LanguageModel_4score(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self, PyObject *__pyx_v_sentence, PyObject *__pyx_v_bos, PyObject *__pyx_v_eos); /* proto */
-static PyObject *__pyx_pf_5kenlm_13LanguageModel_6full_scores(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self, PyObject *__pyx_v_sentence, PyObject *__pyx_v_bos, PyObject *__pyx_v_eos); /* proto */
-static int __pyx_pf_5kenlm_13LanguageModel_9__contains__(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self, PyObject *__pyx_v_word); /* proto */
-static PyObject *__pyx_pf_5kenlm_13LanguageModel_11__repr__(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self); /* proto */
-static PyObject *__pyx_pf_5kenlm_13LanguageModel_13__reduce__(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self); /* proto */
-static PyObject *__pyx_pf_5kenlm_13LanguageModel_4path___get__(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self); /* proto */
-static int __pyx_pf_5kenlm_13LanguageModel_4path_2__set__(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self, PyObject *__pyx_v_value); /* proto */
-static int __pyx_pf_5kenlm_13LanguageModel_4path_4__del__(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self); /* proto */
-static int __pyx_pf_5kenlm_5Model___init__(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_path); /* proto */
-static void __pyx_pf_5kenlm_5Model_2__dealloc__(struct __pyx_obj_5kenlm_Model *__pyx_v_self); /* proto */
-static PyObject *__pyx_pf_5kenlm_5Model_5order___get__(struct __pyx_obj_5kenlm_Model *__pyx_v_self); /* proto */
-static PyObject *__pyx_pf_5kenlm_5Model_4BeginSentenceWrite(struct __pyx_obj_5kenlm_Model *__pyx_v_self, struct __pyx_obj_5kenlm_State *__pyx_v_state); /* proto */
-static PyObject *__pyx_pf_5kenlm_5Model_6NullContextWrite(struct __pyx_obj_5kenlm_Model *__pyx_v_self, struct __pyx_obj_5kenlm_State *__pyx_v_state); /* proto */
-static PyObject *__pyx_pf_5kenlm_5Model_8BaseScore(struct __pyx_obj_5kenlm_Model *__pyx_v_self, struct __pyx_obj_5kenlm_State *__pyx_v_in_state, PyObject *__pyx_v_word, struct __pyx_obj_5kenlm_State *__pyx_v_out_state); /* proto */
-static PyObject *__pyx_pf_5kenlm_5Model_10BaseFullScore(struct __pyx_obj_5kenlm_Model *__pyx_v_self, struct __pyx_obj_5kenlm_State *__pyx_v_in_state, PyObject *__pyx_v_word, struct __pyx_obj_5kenlm_State *__pyx_v_out_state); /* proto */
-static int __pyx_pf_5kenlm_5Model_12__contains__(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_word); /* proto */
-static PyObject *__pyx_pf_5kenlm_5Model_14__repr__(struct __pyx_obj_5kenlm_Model *__pyx_v_self); /* proto */
-static PyObject *__pyx_pf_5kenlm_5Model_16__reduce__(struct __pyx_obj_5kenlm_Model *__pyx_v_self); /* proto */
-static PyObject *__pyx_pf_5kenlm_5Model_4path___get__(struct __pyx_obj_5kenlm_Model *__pyx_v_self); /* proto */
-static int __pyx_pf_5kenlm_5Model_4path_2__set__(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_value); /* proto */
-static int __pyx_pf_5kenlm_5Model_4path_4__del__(struct __pyx_obj_5kenlm_Model *__pyx_v_self); /* proto */
-static PyObject *__pyx_tp_new_5kenlm_FullScoreReturn(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/
-static PyObject *__pyx_tp_new_5kenlm_State(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/
-static PyObject *__pyx_tp_new_5kenlm_LanguageModel(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/
-static PyObject *__pyx_tp_new_5kenlm_Model(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/
-static PyObject *__pyx_tp_new_5kenlm___pyx_scope_struct__full_scores(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/
-static char __pyx_k__2[] = "\n";
-static char __pyx_k__3[] = " ";
-static char __pyx_k_os[] = "os";
-static char __pyx_k_bos[] = "bos";
-static char __pyx_k_eos[] = "eos";
-static char __pyx_k_oov[] = "oov";
-static char __pyx_k_args[] = "args";
-static char __pyx_k_main[] = "__main__";
-static char __pyx_k_name[] = "__name__";
-static char __pyx_k_path[] = "path";
-static char __pyx_k_send[] = "send";
-static char __pyx_k_test[] = "__test__";
-static char __pyx_k_utf8[] = "utf8";
-static char __pyx_k_word[] = "word";
-static char __pyx_k_class[] = "__class__";
-static char __pyx_k_close[] = "close";
-static char __pyx_k_kenlm[] = "_kenlm";
-static char __pyx_k_split[] = "split";
-static char __pyx_k_throw[] = "throw";
-static char __pyx_k_encode[] = "encode";
-static char __pyx_k_format[] = "format";
-static char __pyx_k_import[] = "__import__";
-static char __pyx_k_0_1_2_3[] = "{0}({1}, {2}, {3})";
-static char __pyx_k_IOError[] = "IOError";
-static char __pyx_k_abspath[] = "abspath";
-static char __pyx_k_replace[] = "replace";
-static char __pyx_k_basename[] = "basename";
-static char __pyx_k_in_state[] = "in_state";
-static char __pyx_k_log_prob[] = "log_prob";
-static char __pyx_k_sentence[] = "sentence";
-static char __pyx_k_TypeError[] = "TypeError";
-static char __pyx_k_out_state[] = "out_state";
-static char __pyx_k_full_scores[] = "full_scores";
-static char __pyx_k_Model_from_0[] = "<Model from {0}>";
-static char __pyx_k_RuntimeError[] = "RuntimeError";
-static char __pyx_k_ngram_length[] = "ngram_length";
-static char __pyx_k_LanguageModel[] = "LanguageModel";
-static char __pyx_k_Cannot_read_model[] = "Cannot read model '{}' ({})";
-static char __pyx_k_LanguageModel_from_0[] = "<LanguageModel from {0}>";
-static char __pyx_k_LanguageModel_full_scores[] = "LanguageModel.full_scores";
-static char __pyx_k_Cannot_convert_s_to_string[] = "Cannot convert %s to string";
-static PyObject *__pyx_kp_s_0_1_2_3;
-static PyObject *__pyx_kp_s_Cannot_convert_s_to_string;
-static PyObject *__pyx_kp_s_Cannot_read_model;
-static PyObject *__pyx_n_s_IOError;
-static PyObject *__pyx_n_s_LanguageModel;
-static PyObject *__pyx_kp_s_LanguageModel_from_0;
-static PyObject *__pyx_n_s_LanguageModel_full_scores;
-static PyObject *__pyx_kp_s_Model_from_0;
-static PyObject *__pyx_n_s_RuntimeError;
-static PyObject *__pyx_n_s_TypeError;
-static PyObject *__pyx_kp_s__2;
-static PyObject *__pyx_kp_s__3;
-static PyObject *__pyx_n_s_abspath;
-static PyObject *__pyx_n_s_args;
-static PyObject *__pyx_n_s_basename;
-static PyObject *__pyx_n_s_bos;
-static PyObject *__pyx_n_s_class;
-static PyObject *__pyx_n_s_close;
-static PyObject *__pyx_n_s_encode;
-static PyObject *__pyx_n_s_eos;
-static PyObject *__pyx_n_s_format;
-static PyObject *__pyx_n_s_full_scores;
-static PyObject *__pyx_n_s_import;
-static PyObject *__pyx_n_s_in_state;
-static PyObject *__pyx_n_s_kenlm;
-static PyObject *__pyx_n_s_log_prob;
-static PyObject *__pyx_n_s_main;
-static PyObject *__pyx_n_s_name;
-static PyObject *__pyx_n_s_ngram_length;
-static PyObject *__pyx_n_s_oov;
-static PyObject *__pyx_n_s_os;
-static PyObject *__pyx_n_s_out_state;
-static PyObject *__pyx_n_s_path;
-static PyObject *__pyx_n_s_replace;
-static PyObject *__pyx_n_s_send;
-static PyObject *__pyx_n_s_sentence;
-static PyObject *__pyx_n_s_split;
-static PyObject *__pyx_n_s_test;
-static PyObject *__pyx_n_s_throw;
-static PyObject *__pyx_n_s_utf8;
-static PyObject *__pyx_n_s_word;
-static PyObject *__pyx_tuple_;
-static PyObject *__pyx_tuple__4;
-static PyObject *__pyx_tuple__5;
-
-/* "kenlm.pyx":4
- * cimport _kenlm
- *
- * cdef bytes as_str(data): # <<<<<<<<<<<<<<
- * if isinstance(data, bytes):
- * return data
- */
-
-static PyObject *__pyx_f_5kenlm_as_str(PyObject *__pyx_v_data) {
- PyObject *__pyx_r = NULL;
- __Pyx_RefNannyDeclarations
- int __pyx_t_1;
- int __pyx_t_2;
- PyObject *__pyx_t_3 = NULL;
- PyObject *__pyx_t_4 = NULL;
- int __pyx_lineno = 0;
- const char *__pyx_filename = NULL;
- int __pyx_clineno = 0;
- __Pyx_RefNannySetupContext("as_str", 0);
-
- /* "kenlm.pyx":5
- *
- * cdef bytes as_str(data):
- * if isinstance(data, bytes): # <<<<<<<<<<<<<<
- * return data
- * elif isinstance(data, unicode):
- */
- __pyx_t_1 = PyBytes_Check(__pyx_v_data);
- __pyx_t_2 = (__pyx_t_1 != 0);
- if (__pyx_t_2) {
-
- /* "kenlm.pyx":6
- * cdef bytes as_str(data):
- * if isinstance(data, bytes):
- * return data # <<<<<<<<<<<<<<
- * elif isinstance(data, unicode):
- * return data.encode('utf8')
- */
- __Pyx_XDECREF(__pyx_r);
- if (!(likely(PyBytes_CheckExact(__pyx_v_data))||((__pyx_v_data) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_v_data)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 6; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_INCREF(__pyx_v_data);
- __pyx_r = ((PyObject*)__pyx_v_data);
- goto __pyx_L0;
- }
-
- /* "kenlm.pyx":7
- * if isinstance(data, bytes):
- * return data
- * elif isinstance(data, unicode): # <<<<<<<<<<<<<<
- * return data.encode('utf8')
- * raise TypeError('Cannot convert %s to string' % type(data))
- */
- __pyx_t_2 = PyUnicode_Check(__pyx_v_data);
- __pyx_t_1 = (__pyx_t_2 != 0);
- if (__pyx_t_1) {
-
- /* "kenlm.pyx":8
- * return data
- * elif isinstance(data, unicode):
- * return data.encode('utf8') # <<<<<<<<<<<<<<
- * raise TypeError('Cannot convert %s to string' % type(data))
- *
- */
- __Pyx_XDECREF(__pyx_r);
- __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_data, __pyx_n_s_encode); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 8; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_3);
- __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 8; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_4);
- __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
- if (!(likely(PyBytes_CheckExact(__pyx_t_4))||((__pyx_t_4) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_t_4)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 8; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __pyx_r = ((PyObject*)__pyx_t_4);
- __pyx_t_4 = 0;
- goto __pyx_L0;
- }
-
- /* "kenlm.pyx":9
- * elif isinstance(data, unicode):
- * return data.encode('utf8')
- * raise TypeError('Cannot convert %s to string' % type(data)) # <<<<<<<<<<<<<<
- *
- * cdef class FullScoreReturn:
- */
- __pyx_t_4 = __Pyx_PyString_Format(__pyx_kp_s_Cannot_convert_s_to_string, ((PyObject *)Py_TYPE(__pyx_v_data))); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 9; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_4);
- __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 9; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_3);
- PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_4);
- __Pyx_GIVEREF(__pyx_t_4);
- __pyx_t_4 = 0;
- __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_t_3, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 9; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_4);
- __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
- __Pyx_Raise(__pyx_t_4, 0, 0, 0);
- __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
- {__pyx_filename = __pyx_f[0]; __pyx_lineno = 9; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-
- /* "kenlm.pyx":4
- * cimport _kenlm
- *
- * cdef bytes as_str(data): # <<<<<<<<<<<<<<
- * if isinstance(data, bytes):
- * return data
- */
-
- /* function exit code */
- __pyx_L1_error:;
- __Pyx_XDECREF(__pyx_t_3);
- __Pyx_XDECREF(__pyx_t_4);
- __Pyx_AddTraceback("kenlm.as_str", __pyx_clineno, __pyx_lineno, __pyx_filename);
- __pyx_r = 0;
- __pyx_L0:;
- __Pyx_XGIVEREF(__pyx_r);
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-/* "kenlm.pyx":24
- * cdef bint oov
- *
- * def __cinit__(self, log_prob, ngram_length, oov): # <<<<<<<<<<<<<<
- * self.log_prob = log_prob
- * self.ngram_length = ngram_length
- */
-
-/* Python wrapper */
-static int __pyx_pw_5kenlm_15FullScoreReturn_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
-static int __pyx_pw_5kenlm_15FullScoreReturn_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
- PyObject *__pyx_v_log_prob = 0;
- PyObject *__pyx_v_ngram_length = 0;
- PyObject *__pyx_v_oov = 0;
- int __pyx_lineno = 0;
- const char *__pyx_filename = NULL;
- int __pyx_clineno = 0;
- int __pyx_r;
- __Pyx_RefNannyDeclarations
- __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0);
- {
- static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_log_prob,&__pyx_n_s_ngram_length,&__pyx_n_s_oov,0};
- PyObject* values[3] = {0,0,0};
- if (unlikely(__pyx_kwds)) {
- Py_ssize_t kw_args;
- const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args);
- switch (pos_args) {
- case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2);
- case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1);
- case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
- case 0: break;
- default: goto __pyx_L5_argtuple_error;
- }
- kw_args = PyDict_Size(__pyx_kwds);
- switch (pos_args) {
- case 0:
- if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_log_prob)) != 0)) kw_args--;
- else goto __pyx_L5_argtuple_error;
- case 1:
- if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_ngram_length)) != 0)) kw_args--;
- else {
- __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 3, 3, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 24; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
- }
- case 2:
- if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_oov)) != 0)) kw_args--;
- else {
- __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 3, 3, 2); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 24; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
- }
- }
- if (unlikely(kw_args > 0)) {
- if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 24; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
- }
- } else if (PyTuple_GET_SIZE(__pyx_args) != 3) {
- goto __pyx_L5_argtuple_error;
- } else {
- values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
- values[1] = PyTuple_GET_ITEM(__pyx_args, 1);
- values[2] = PyTuple_GET_ITEM(__pyx_args, 2);
- }
- __pyx_v_log_prob = values[0];
- __pyx_v_ngram_length = values[1];
- __pyx_v_oov = values[2];
- }
- goto __pyx_L4_argument_unpacking_done;
- __pyx_L5_argtuple_error:;
- __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 24; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
- __pyx_L3_error:;
- __Pyx_AddTraceback("kenlm.FullScoreReturn.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename);
- __Pyx_RefNannyFinishContext();
- return -1;
- __pyx_L4_argument_unpacking_done:;
- __pyx_r = __pyx_pf_5kenlm_15FullScoreReturn___cinit__(((struct __pyx_obj_5kenlm_FullScoreReturn *)__pyx_v_self), __pyx_v_log_prob, __pyx_v_ngram_length, __pyx_v_oov);
-
- /* function exit code */
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-static int __pyx_pf_5kenlm_15FullScoreReturn___cinit__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self, PyObject *__pyx_v_log_prob, PyObject *__pyx_v_ngram_length, PyObject *__pyx_v_oov) {
- int __pyx_r;
- __Pyx_RefNannyDeclarations
- float __pyx_t_1;
- int __pyx_t_2;
- int __pyx_t_3;
- int __pyx_lineno = 0;
- const char *__pyx_filename = NULL;
- int __pyx_clineno = 0;
- __Pyx_RefNannySetupContext("__cinit__", 0);
-
- /* "kenlm.pyx":25
- *
- * def __cinit__(self, log_prob, ngram_length, oov):
- * self.log_prob = log_prob # <<<<<<<<<<<<<<
- * self.ngram_length = ngram_length
- * self.oov = oov
- */
- __pyx_t_1 = __pyx_PyFloat_AsFloat(__pyx_v_log_prob); if (unlikely((__pyx_t_1 == (float)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 25; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __pyx_v_self->log_prob = __pyx_t_1;
-
- /* "kenlm.pyx":26
- * def __cinit__(self, log_prob, ngram_length, oov):
- * self.log_prob = log_prob
- * self.ngram_length = ngram_length # <<<<<<<<<<<<<<
- * self.oov = oov
- *
- */
- __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_v_ngram_length); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 26; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __pyx_v_self->ngram_length = __pyx_t_2;
-
- /* "kenlm.pyx":27
- * self.log_prob = log_prob
- * self.ngram_length = ngram_length
- * self.oov = oov # <<<<<<<<<<<<<<
- *
- * def __repr__(self):
- */
- __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_v_oov); if (unlikely((__pyx_t_3 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 27; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __pyx_v_self->oov = __pyx_t_3;
-
- /* "kenlm.pyx":24
- * cdef bint oov
- *
- * def __cinit__(self, log_prob, ngram_length, oov): # <<<<<<<<<<<<<<
- * self.log_prob = log_prob
- * self.ngram_length = ngram_length
- */
-
- /* function exit code */
- __pyx_r = 0;
- goto __pyx_L0;
- __pyx_L1_error:;
- __Pyx_AddTraceback("kenlm.FullScoreReturn.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename);
- __pyx_r = -1;
- __pyx_L0:;
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-/* "kenlm.pyx":29
- * self.oov = oov
- *
- * def __repr__(self): # <<<<<<<<<<<<<<
- * return '{0}({1}, {2}, {3})'.format(self.__class__.__name__, repr(self.log_prob), repr(self.ngram_length), repr(self.oov))
- *
- */
-
-/* Python wrapper */
-static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_3__repr__(PyObject *__pyx_v_self); /*proto*/
-static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_3__repr__(PyObject *__pyx_v_self) {
- PyObject *__pyx_r = 0;
- __Pyx_RefNannyDeclarations
- __Pyx_RefNannySetupContext("__repr__ (wrapper)", 0);
- __pyx_r = __pyx_pf_5kenlm_15FullScoreReturn_2__repr__(((struct __pyx_obj_5kenlm_FullScoreReturn *)__pyx_v_self));
-
- /* function exit code */
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_2__repr__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self) {
- PyObject *__pyx_r = NULL;
- __Pyx_RefNannyDeclarations
- PyObject *__pyx_t_1 = NULL;
- PyObject *__pyx_t_2 = NULL;
- PyObject *__pyx_t_3 = NULL;
- PyObject *__pyx_t_4 = NULL;
- PyObject *__pyx_t_5 = NULL;
- PyObject *__pyx_t_6 = NULL;
- PyObject *__pyx_t_7 = NULL;
- Py_ssize_t __pyx_t_8;
- PyObject *__pyx_t_9 = NULL;
- int __pyx_lineno = 0;
- const char *__pyx_filename = NULL;
- int __pyx_clineno = 0;
- __Pyx_RefNannySetupContext("__repr__", 0);
-
- /* "kenlm.pyx":30
- *
- * def __repr__(self):
- * return '{0}({1}, {2}, {3})'.format(self.__class__.__name__, repr(self.log_prob), repr(self.ngram_length), repr(self.oov)) # <<<<<<<<<<<<<<
- *
- * property log_prob:
- */
- __Pyx_XDECREF(__pyx_r);
- __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_kp_s_0_1_2_3, __pyx_n_s_format); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_2);
- __pyx_t_3 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_n_s_class); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_3);
- __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_name); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_4);
- __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
- __pyx_t_3 = PyFloat_FromDouble(__pyx_v_self->log_prob); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_3);
- __pyx_t_5 = PyObject_Repr(__pyx_t_3); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_5);
- __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
- __pyx_t_3 = __Pyx_PyInt_From_int(__pyx_v_self->ngram_length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_3);
- __pyx_t_6 = PyObject_Repr(__pyx_t_3); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_6);
- __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
- __pyx_t_3 = __Pyx_PyBool_FromLong(__pyx_v_self->oov); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_3);
- __pyx_t_7 = PyObject_Repr(__pyx_t_3); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_7);
- __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
- __pyx_t_3 = NULL;
- __pyx_t_8 = 0;
- if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_2))) {
- __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_2);
- if (likely(__pyx_t_3)) {
- PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2);
- __Pyx_INCREF(__pyx_t_3);
- __Pyx_INCREF(function);
- __Pyx_DECREF_SET(__pyx_t_2, function);
- __pyx_t_8 = 1;
- }
- }
- __pyx_t_9 = PyTuple_New(4+__pyx_t_8); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_9);
- if (__pyx_t_3) {
- PyTuple_SET_ITEM(__pyx_t_9, 0, __pyx_t_3); __Pyx_GIVEREF(__pyx_t_3); __pyx_t_3 = NULL;
- }
- PyTuple_SET_ITEM(__pyx_t_9, 0+__pyx_t_8, __pyx_t_4);
- __Pyx_GIVEREF(__pyx_t_4);
- PyTuple_SET_ITEM(__pyx_t_9, 1+__pyx_t_8, __pyx_t_5);
- __Pyx_GIVEREF(__pyx_t_5);
- PyTuple_SET_ITEM(__pyx_t_9, 2+__pyx_t_8, __pyx_t_6);
- __Pyx_GIVEREF(__pyx_t_6);
- PyTuple_SET_ITEM(__pyx_t_9, 3+__pyx_t_8, __pyx_t_7);
- __Pyx_GIVEREF(__pyx_t_7);
- __pyx_t_4 = 0;
- __pyx_t_5 = 0;
- __pyx_t_6 = 0;
- __pyx_t_7 = 0;
- __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_9, NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_1);
- __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
- __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
- __pyx_r = __pyx_t_1;
- __pyx_t_1 = 0;
- goto __pyx_L0;
-
- /* "kenlm.pyx":29
- * self.oov = oov
- *
- * def __repr__(self): # <<<<<<<<<<<<<<
- * return '{0}({1}, {2}, {3})'.format(self.__class__.__name__, repr(self.log_prob), repr(self.ngram_length), repr(self.oov))
- *
- */
-
- /* function exit code */
- __pyx_L1_error:;
- __Pyx_XDECREF(__pyx_t_1);
- __Pyx_XDECREF(__pyx_t_2);
- __Pyx_XDECREF(__pyx_t_3);
- __Pyx_XDECREF(__pyx_t_4);
- __Pyx_XDECREF(__pyx_t_5);
- __Pyx_XDECREF(__pyx_t_6);
- __Pyx_XDECREF(__pyx_t_7);
- __Pyx_XDECREF(__pyx_t_9);
- __Pyx_AddTraceback("kenlm.FullScoreReturn.__repr__", __pyx_clineno, __pyx_lineno, __pyx_filename);
- __pyx_r = NULL;
- __pyx_L0:;
- __Pyx_XGIVEREF(__pyx_r);
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-/* "kenlm.pyx":33
- *
- * property log_prob:
- * def __get__(self): # <<<<<<<<<<<<<<
- * return self.log_prob
- *
- */
-
-/* Python wrapper */
-static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_8log_prob_1__get__(PyObject *__pyx_v_self); /*proto*/
-static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_8log_prob_1__get__(PyObject *__pyx_v_self) {
- PyObject *__pyx_r = 0;
- __Pyx_RefNannyDeclarations
- __Pyx_RefNannySetupContext("__get__ (wrapper)", 0);
- __pyx_r = __pyx_pf_5kenlm_15FullScoreReturn_8log_prob___get__(((struct __pyx_obj_5kenlm_FullScoreReturn *)__pyx_v_self));
-
- /* function exit code */
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_8log_prob___get__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self) {
- PyObject *__pyx_r = NULL;
- __Pyx_RefNannyDeclarations
- PyObject *__pyx_t_1 = NULL;
- int __pyx_lineno = 0;
- const char *__pyx_filename = NULL;
- int __pyx_clineno = 0;
- __Pyx_RefNannySetupContext("__get__", 0);
-
- /* "kenlm.pyx":34
- * property log_prob:
- * def __get__(self):
- * return self.log_prob # <<<<<<<<<<<<<<
- *
- * property ngram_length:
- */
- __Pyx_XDECREF(__pyx_r);
- __pyx_t_1 = PyFloat_FromDouble(__pyx_v_self->log_prob); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 34; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_1);
- __pyx_r = __pyx_t_1;
- __pyx_t_1 = 0;
- goto __pyx_L0;
-
- /* "kenlm.pyx":33
- *
- * property log_prob:
- * def __get__(self): # <<<<<<<<<<<<<<
- * return self.log_prob
- *
- */
-
- /* function exit code */
- __pyx_L1_error:;
- __Pyx_XDECREF(__pyx_t_1);
- __Pyx_AddTraceback("kenlm.FullScoreReturn.log_prob.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename);
- __pyx_r = NULL;
- __pyx_L0:;
- __Pyx_XGIVEREF(__pyx_r);
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-/* "kenlm.pyx":37
- *
- * property ngram_length:
- * def __get__(self): # <<<<<<<<<<<<<<
- * return self.ngram_length
- *
- */
-
-/* Python wrapper */
-static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_12ngram_length_1__get__(PyObject *__pyx_v_self); /*proto*/
-static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_12ngram_length_1__get__(PyObject *__pyx_v_self) {
- PyObject *__pyx_r = 0;
- __Pyx_RefNannyDeclarations
- __Pyx_RefNannySetupContext("__get__ (wrapper)", 0);
- __pyx_r = __pyx_pf_5kenlm_15FullScoreReturn_12ngram_length___get__(((struct __pyx_obj_5kenlm_FullScoreReturn *)__pyx_v_self));
-
- /* function exit code */
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_12ngram_length___get__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self) {
- PyObject *__pyx_r = NULL;
- __Pyx_RefNannyDeclarations
- PyObject *__pyx_t_1 = NULL;
- int __pyx_lineno = 0;
- const char *__pyx_filename = NULL;
- int __pyx_clineno = 0;
- __Pyx_RefNannySetupContext("__get__", 0);
-
- /* "kenlm.pyx":38
- * property ngram_length:
- * def __get__(self):
- * return self.ngram_length # <<<<<<<<<<<<<<
- *
- * property oov:
- */
- __Pyx_XDECREF(__pyx_r);
- __pyx_t_1 = __Pyx_PyInt_From_int(__pyx_v_self->ngram_length); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 38; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_1);
- __pyx_r = __pyx_t_1;
- __pyx_t_1 = 0;
- goto __pyx_L0;
-
- /* "kenlm.pyx":37
- *
- * property ngram_length:
- * def __get__(self): # <<<<<<<<<<<<<<
- * return self.ngram_length
- *
- */
-
- /* function exit code */
- __pyx_L1_error:;
- __Pyx_XDECREF(__pyx_t_1);
- __Pyx_AddTraceback("kenlm.FullScoreReturn.ngram_length.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename);
- __pyx_r = NULL;
- __pyx_L0:;
- __Pyx_XGIVEREF(__pyx_r);
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-/* "kenlm.pyx":41
- *
- * property oov:
- * def __get__(self): # <<<<<<<<<<<<<<
- * return self.oov
- *
- */
-
-/* Python wrapper */
-static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_3oov_1__get__(PyObject *__pyx_v_self); /*proto*/
-static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_3oov_1__get__(PyObject *__pyx_v_self) {
- PyObject *__pyx_r = 0;
- __Pyx_RefNannyDeclarations
- __Pyx_RefNannySetupContext("__get__ (wrapper)", 0);
- __pyx_r = __pyx_pf_5kenlm_15FullScoreReturn_3oov___get__(((struct __pyx_obj_5kenlm_FullScoreReturn *)__pyx_v_self));
-
- /* function exit code */
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_3oov___get__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self) {
- PyObject *__pyx_r = NULL;
- __Pyx_RefNannyDeclarations
- PyObject *__pyx_t_1 = NULL;
- int __pyx_lineno = 0;
- const char *__pyx_filename = NULL;
- int __pyx_clineno = 0;
- __Pyx_RefNannySetupContext("__get__", 0);
-
- /* "kenlm.pyx":42
- * property oov:
- * def __get__(self):
- * return self.oov # <<<<<<<<<<<<<<
- *
- * cdef class State:
- */
- __Pyx_XDECREF(__pyx_r);
- __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->oov); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 42; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_1);
- __pyx_r = __pyx_t_1;
- __pyx_t_1 = 0;
- goto __pyx_L0;
-
- /* "kenlm.pyx":41
- *
- * property oov:
- * def __get__(self): # <<<<<<<<<<<<<<
- * return self.oov
- *
- */
-
- /* function exit code */
- __pyx_L1_error:;
- __Pyx_XDECREF(__pyx_t_1);
- __Pyx_AddTraceback("kenlm.FullScoreReturn.oov.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename);
- __pyx_r = NULL;
- __pyx_L0:;
- __Pyx_XGIVEREF(__pyx_r);
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-/* "kenlm.pyx":55
- * cdef _kenlm.State _c_state
- *
- * def __richcmp__(State qa, State qb, int op): # <<<<<<<<<<<<<<
- * r = qa._c_state.Compare(qb._c_state)
- * if op == 0: # <
- */
-
-/* Python wrapper */
-static PyObject *__pyx_pw_5kenlm_5State_1__richcmp__(PyObject *__pyx_v_qa, PyObject *__pyx_v_qb, int __pyx_v_op); /*proto*/
-static PyObject *__pyx_pw_5kenlm_5State_1__richcmp__(PyObject *__pyx_v_qa, PyObject *__pyx_v_qb, int __pyx_v_op) {
- CYTHON_UNUSED int __pyx_lineno = 0;
- CYTHON_UNUSED const char *__pyx_filename = NULL;
- CYTHON_UNUSED int __pyx_clineno = 0;
- PyObject *__pyx_r = 0;
- __Pyx_RefNannyDeclarations
- __Pyx_RefNannySetupContext("__richcmp__ (wrapper)", 0);
- if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_qa), __pyx_ptype_5kenlm_State, 1, "qa", 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 55; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_qb), __pyx_ptype_5kenlm_State, 1, "qb", 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 55; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __pyx_r = __pyx_pf_5kenlm_5State___richcmp__(((struct __pyx_obj_5kenlm_State *)__pyx_v_qa), ((struct __pyx_obj_5kenlm_State *)__pyx_v_qb), ((int)__pyx_v_op));
-
- /* function exit code */
- goto __pyx_L0;
- __pyx_L1_error:;
- __pyx_r = NULL;
- __pyx_L0:;
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-static PyObject *__pyx_pf_5kenlm_5State___richcmp__(struct __pyx_obj_5kenlm_State *__pyx_v_qa, struct __pyx_obj_5kenlm_State *__pyx_v_qb, int __pyx_v_op) {
- int __pyx_v_r;
- PyObject *__pyx_r = NULL;
- __Pyx_RefNannyDeclarations
- PyObject *__pyx_t_1 = NULL;
- int __pyx_lineno = 0;
- const char *__pyx_filename = NULL;
- int __pyx_clineno = 0;
- __Pyx_RefNannySetupContext("__richcmp__", 0);
-
- /* "kenlm.pyx":56
- *
- * def __richcmp__(State qa, State qb, int op):
- * r = qa._c_state.Compare(qb._c_state) # <<<<<<<<<<<<<<
- * if op == 0: # <
- * return r < 0
- */
- __pyx_v_r = __pyx_v_qa->_c_state.Compare(__pyx_v_qb->_c_state);
-
- /* "kenlm.pyx":65
- * elif op == 3: # !=
- * return r != 0
- * elif op == 4: # > # <<<<<<<<<<<<<<
- * return r > 0
- * else: # >=
- */
- switch (__pyx_v_op) {
-
- /* "kenlm.pyx":57
- * def __richcmp__(State qa, State qb, int op):
- * r = qa._c_state.Compare(qb._c_state)
- * if op == 0: # < # <<<<<<<<<<<<<<
- * return r < 0
- * elif op == 1: # <=
- */
- case 0:
-
- /* "kenlm.pyx":58
- * r = qa._c_state.Compare(qb._c_state)
- * if op == 0: # <
- * return r < 0 # <<<<<<<<<<<<<<
- * elif op == 1: # <=
- * return r <= 0
- */
- __Pyx_XDECREF(__pyx_r);
- __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_r < 0)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 58; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_1);
- __pyx_r = __pyx_t_1;
- __pyx_t_1 = 0;
- goto __pyx_L0;
- break;
-
- /* "kenlm.pyx":59
- * if op == 0: # <
- * return r < 0
- * elif op == 1: # <= # <<<<<<<<<<<<<<
- * return r <= 0
- * elif op == 2: # ==
- */
- case 1:
-
- /* "kenlm.pyx":60
- * return r < 0
- * elif op == 1: # <=
- * return r <= 0 # <<<<<<<<<<<<<<
- * elif op == 2: # ==
- * return r == 0
- */
- __Pyx_XDECREF(__pyx_r);
- __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_r <= 0)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 60; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_1);
- __pyx_r = __pyx_t_1;
- __pyx_t_1 = 0;
- goto __pyx_L0;
- break;
-
- /* "kenlm.pyx":61
- * elif op == 1: # <=
- * return r <= 0
- * elif op == 2: # == # <<<<<<<<<<<<<<
- * return r == 0
- * elif op == 3: # !=
- */
- case 2:
-
- /* "kenlm.pyx":62
- * return r <= 0
- * elif op == 2: # ==
- * return r == 0 # <<<<<<<<<<<<<<
- * elif op == 3: # !=
- * return r != 0
- */
- __Pyx_XDECREF(__pyx_r);
- __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_r == 0)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_1);
- __pyx_r = __pyx_t_1;
- __pyx_t_1 = 0;
- goto __pyx_L0;
- break;
-
- /* "kenlm.pyx":63
- * elif op == 2: # ==
- * return r == 0
- * elif op == 3: # != # <<<<<<<<<<<<<<
- * return r != 0
- * elif op == 4: # >
- */
- case 3:
-
- /* "kenlm.pyx":64
- * return r == 0
- * elif op == 3: # !=
- * return r != 0 # <<<<<<<<<<<<<<
- * elif op == 4: # >
- * return r > 0
- */
- __Pyx_XDECREF(__pyx_r);
- __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_r != 0)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 64; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_1);
- __pyx_r = __pyx_t_1;
- __pyx_t_1 = 0;
- goto __pyx_L0;
- break;
-
- /* "kenlm.pyx":65
- * elif op == 3: # !=
- * return r != 0
- * elif op == 4: # > # <<<<<<<<<<<<<<
- * return r > 0
- * else: # >=
- */
- case 4:
-
- /* "kenlm.pyx":66
- * return r != 0
- * elif op == 4: # >
- * return r > 0 # <<<<<<<<<<<<<<
- * else: # >=
- * return r >= 0
- */
- __Pyx_XDECREF(__pyx_r);
- __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_r > 0)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_1);
- __pyx_r = __pyx_t_1;
- __pyx_t_1 = 0;
- goto __pyx_L0;
- break;
- default:
-
- /* "kenlm.pyx":68
- * return r > 0
- * else: # >=
- * return r >= 0 # <<<<<<<<<<<<<<
- *
- * def __hash__(self):
- */
- __Pyx_XDECREF(__pyx_r);
- __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_r >= 0)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_1);
- __pyx_r = __pyx_t_1;
- __pyx_t_1 = 0;
- goto __pyx_L0;
- break;
- }
-
- /* "kenlm.pyx":55
- * cdef _kenlm.State _c_state
- *
- * def __richcmp__(State qa, State qb, int op): # <<<<<<<<<<<<<<
- * r = qa._c_state.Compare(qb._c_state)
- * if op == 0: # <
- */
-
- /* function exit code */
- __pyx_L1_error:;
- __Pyx_XDECREF(__pyx_t_1);
- __Pyx_AddTraceback("kenlm.State.__richcmp__", __pyx_clineno, __pyx_lineno, __pyx_filename);
- __pyx_r = NULL;
- __pyx_L0:;
- __Pyx_XGIVEREF(__pyx_r);
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-/* "kenlm.pyx":70
- * return r >= 0
- *
- * def __hash__(self): # <<<<<<<<<<<<<<
- * return _kenlm.hash_value(self._c_state)
- *
- */
-
-/* Python wrapper */
-static Py_hash_t __pyx_pw_5kenlm_5State_3__hash__(PyObject *__pyx_v_self); /*proto*/
-static Py_hash_t __pyx_pw_5kenlm_5State_3__hash__(PyObject *__pyx_v_self) {
- Py_hash_t __pyx_r;
- __Pyx_RefNannyDeclarations
- __Pyx_RefNannySetupContext("__hash__ (wrapper)", 0);
- __pyx_r = __pyx_pf_5kenlm_5State_2__hash__(((struct __pyx_obj_5kenlm_State *)__pyx_v_self));
-
- /* function exit code */
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-static Py_hash_t __pyx_pf_5kenlm_5State_2__hash__(struct __pyx_obj_5kenlm_State *__pyx_v_self) {
- Py_hash_t __pyx_r;
- __Pyx_RefNannyDeclarations
- __Pyx_RefNannySetupContext("__hash__", 0);
-
- /* "kenlm.pyx":71
- *
- * def __hash__(self):
- * return _kenlm.hash_value(self._c_state) # <<<<<<<<<<<<<<
- *
- *
- */
- __pyx_r = lm::ngram::hash_value(__pyx_v_self->_c_state);
- goto __pyx_L0;
-
- /* "kenlm.pyx":70
- * return r >= 0
- *
- * def __hash__(self): # <<<<<<<<<<<<<<
- * return _kenlm.hash_value(self._c_state)
- *
- */
-
- /* function exit code */
- __pyx_L0:;
- if (unlikely(__pyx_r == -1) && !PyErr_Occurred()) __pyx_r = -2;
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-/* "kenlm.pyx":84
- * cdef _kenlm.const_Vocabulary* vocab
- *
- * def __init__(self, path): # <<<<<<<<<<<<<<
- * """
- * Load the language model.
- */
-
-/* Python wrapper */
-static int __pyx_pw_5kenlm_13LanguageModel_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
-static char __pyx_doc_5kenlm_13LanguageModel___init__[] = "\n Load the language model.\n\n :param path: path to an arpa file or a kenlm binary file.\n ";
-#if CYTHON_COMPILING_IN_CPYTHON
-struct wrapperbase __pyx_wrapperbase_5kenlm_13LanguageModel___init__;
-#endif
-static int __pyx_pw_5kenlm_13LanguageModel_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
- PyObject *__pyx_v_path = 0;
- int __pyx_lineno = 0;
- const char *__pyx_filename = NULL;
- int __pyx_clineno = 0;
- int __pyx_r;
- __Pyx_RefNannyDeclarations
- __Pyx_RefNannySetupContext("__init__ (wrapper)", 0);
- {
- static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_path,0};
- PyObject* values[1] = {0};
- if (unlikely(__pyx_kwds)) {
- Py_ssize_t kw_args;
- const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args);
- switch (pos_args) {
- case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
- case 0: break;
- default: goto __pyx_L5_argtuple_error;
- }
- kw_args = PyDict_Size(__pyx_kwds);
- switch (pos_args) {
- case 0:
- if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_path)) != 0)) kw_args--;
- else goto __pyx_L5_argtuple_error;
- }
- if (unlikely(kw_args > 0)) {
- if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
- }
- } else if (PyTuple_GET_SIZE(__pyx_args) != 1) {
- goto __pyx_L5_argtuple_error;
- } else {
- values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
- }
- __pyx_v_path = values[0];
- }
- goto __pyx_L4_argument_unpacking_done;
- __pyx_L5_argtuple_error:;
- __Pyx_RaiseArgtupleInvalid("__init__", 1, 1, 1, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
- __pyx_L3_error:;
- __Pyx_AddTraceback("kenlm.LanguageModel.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename);
- __Pyx_RefNannyFinishContext();
- return -1;
- __pyx_L4_argument_unpacking_done:;
- __pyx_r = __pyx_pf_5kenlm_13LanguageModel___init__(((struct __pyx_obj_5kenlm_LanguageModel *)__pyx_v_self), __pyx_v_path);
-
- /* function exit code */
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-static int __pyx_pf_5kenlm_13LanguageModel___init__(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self, PyObject *__pyx_v_path) {
- PyObject *__pyx_v_exception = NULL;
- PyObject *__pyx_v_exception_message = NULL;
- int __pyx_r;
- __Pyx_RefNannyDeclarations
- PyObject *__pyx_t_1 = NULL;
- PyObject *__pyx_t_2 = NULL;
- PyObject *__pyx_t_3 = NULL;
- PyObject *__pyx_t_4 = NULL;
- PyObject *__pyx_t_5 = NULL;
- PyObject *__pyx_t_6 = NULL;
- PyObject *__pyx_t_7 = NULL;
- PyObject *__pyx_t_8 = NULL;
- char *__pyx_t_9;
- lm::base::Model *__pyx_t_10;
- int __pyx_t_11;
- PyObject *__pyx_t_12 = NULL;
- Py_ssize_t __pyx_t_13;
- PyObject *__pyx_t_14 = NULL;
- int __pyx_lineno = 0;
- const char *__pyx_filename = NULL;
- int __pyx_clineno = 0;
- __Pyx_RefNannySetupContext("__init__", 0);
-
- /* "kenlm.pyx":90
- * :param path: path to an arpa file or a kenlm binary file.
- * """
- * self.path = os.path.abspath(as_str(path)) # <<<<<<<<<<<<<<
- * try:
- * self.model = _kenlm.LoadVirtual(self.path)
- */
- __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_os); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_2);
- __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_path); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_3);
- __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
- __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_abspath); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_2);
- __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
- __pyx_t_3 = __pyx_f_5kenlm_as_str(__pyx_v_path); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_3);
- __pyx_t_4 = NULL;
- if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_2))) {
- __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_2);
- if (likely(__pyx_t_4)) {
- PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2);
- __Pyx_INCREF(__pyx_t_4);
- __Pyx_INCREF(function);
- __Pyx_DECREF_SET(__pyx_t_2, function);
- }
- }
- if (!__pyx_t_4) {
- __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
- __Pyx_GOTREF(__pyx_t_1);
- } else {
- __pyx_t_5 = PyTuple_New(1+1); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_5);
- PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_4); __Pyx_GIVEREF(__pyx_t_4); __pyx_t_4 = NULL;
- PyTuple_SET_ITEM(__pyx_t_5, 0+1, __pyx_t_3);
- __Pyx_GIVEREF(__pyx_t_3);
- __pyx_t_3 = 0;
- __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_1);
- __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
- }
- __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
- if (!(likely(PyBytes_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_t_1)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GIVEREF(__pyx_t_1);
- __Pyx_GOTREF(__pyx_v_self->path);
- __Pyx_DECREF(__pyx_v_self->path);
- __pyx_v_self->path = ((PyObject*)__pyx_t_1);
- __pyx_t_1 = 0;
-
- /* "kenlm.pyx":91
- * """
- * self.path = os.path.abspath(as_str(path))
- * try: # <<<<<<<<<<<<<<
- * self.model = _kenlm.LoadVirtual(self.path)
- * except RuntimeError as exception:
- */
- {
- __Pyx_ExceptionSave(&__pyx_t_6, &__pyx_t_7, &__pyx_t_8);
- __Pyx_XGOTREF(__pyx_t_6);
- __Pyx_XGOTREF(__pyx_t_7);
- __Pyx_XGOTREF(__pyx_t_8);
- /*try:*/ {
-
- /* "kenlm.pyx":92
- * self.path = os.path.abspath(as_str(path))
- * try:
- * self.model = _kenlm.LoadVirtual(self.path) # <<<<<<<<<<<<<<
- * except RuntimeError as exception:
- * exception_message = str(exception).replace('\n', ' ')
- */
- __pyx_t_9 = __Pyx_PyObject_AsString(__pyx_v_self->path); if (unlikely((!__pyx_t_9) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 92; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
- try {
- __pyx_t_10 = lm::ngram::LoadVirtual(__pyx_t_9);
- } catch(...) {
- __Pyx_CppExn2PyErr();
- {__pyx_filename = __pyx_f[0]; __pyx_lineno = 92; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
- }
- __pyx_v_self->model = __pyx_t_10;
- }
- __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0;
- __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0;
- __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0;
- goto __pyx_L10_try_end;
- __pyx_L3_error:;
- __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
- __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
- __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
- __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0;
- __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0;
-
- /* "kenlm.pyx":93
- * try:
- * self.model = _kenlm.LoadVirtual(self.path)
- * except RuntimeError as exception: # <<<<<<<<<<<<<<
- * exception_message = str(exception).replace('\n', ' ')
- * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\
- */
- __pyx_t_11 = PyErr_ExceptionMatches(__pyx_builtin_RuntimeError);
- if (__pyx_t_11) {
- __Pyx_AddTraceback("kenlm.LanguageModel.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename);
- if (__Pyx_GetException(&__pyx_t_1, &__pyx_t_2, &__pyx_t_5) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 93; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;}
- __Pyx_GOTREF(__pyx_t_1);
- __Pyx_GOTREF(__pyx_t_2);
- __Pyx_GOTREF(__pyx_t_5);
- __Pyx_INCREF(__pyx_t_2);
- __pyx_v_exception = __pyx_t_2;
-
- /* "kenlm.pyx":94
- * self.model = _kenlm.LoadVirtual(self.path)
- * except RuntimeError as exception:
- * exception_message = str(exception).replace('\n', ' ') # <<<<<<<<<<<<<<
- * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\
- * from exception
- */
- __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 94; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;}
- __Pyx_GOTREF(__pyx_t_3);
- __Pyx_INCREF(__pyx_v_exception);
- PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_v_exception);
- __Pyx_GIVEREF(__pyx_v_exception);
- __pyx_t_4 = __Pyx_PyObject_Call(((PyObject *)((PyObject*)(&PyString_Type))), __pyx_t_3, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 94; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;}
- __Pyx_GOTREF(__pyx_t_4);
- __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
- __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_replace); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 94; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;}
- __Pyx_GOTREF(__pyx_t_3);
- __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
- __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_tuple__4, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 94; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;}
- __Pyx_GOTREF(__pyx_t_4);
- __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
- __pyx_v_exception_message = __pyx_t_4;
- __pyx_t_4 = 0;
-
- /* "kenlm.pyx":95
- * except RuntimeError as exception:
- * exception_message = str(exception).replace('\n', ' ')
- * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\ # <<<<<<<<<<<<<<
- * from exception
- * self.vocab = &self.model.BaseVocabulary()
- */
- __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_kp_s_Cannot_read_model, __pyx_n_s_format); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;}
- __Pyx_GOTREF(__pyx_t_3);
- __pyx_t_12 = NULL;
- __pyx_t_13 = 0;
- if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_3))) {
- __pyx_t_12 = PyMethod_GET_SELF(__pyx_t_3);
- if (likely(__pyx_t_12)) {
- PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3);
- __Pyx_INCREF(__pyx_t_12);
- __Pyx_INCREF(function);
- __Pyx_DECREF_SET(__pyx_t_3, function);
- __pyx_t_13 = 1;
- }
- }
- __pyx_t_14 = PyTuple_New(2+__pyx_t_13); if (unlikely(!__pyx_t_14)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;}
- __Pyx_GOTREF(__pyx_t_14);
- if (__pyx_t_12) {
- PyTuple_SET_ITEM(__pyx_t_14, 0, __pyx_t_12); __Pyx_GIVEREF(__pyx_t_12); __pyx_t_12 = NULL;
- }
- __Pyx_INCREF(__pyx_v_path);
- PyTuple_SET_ITEM(__pyx_t_14, 0+__pyx_t_13, __pyx_v_path);
- __Pyx_GIVEREF(__pyx_v_path);
- __Pyx_INCREF(__pyx_v_exception_message);
- PyTuple_SET_ITEM(__pyx_t_14, 1+__pyx_t_13, __pyx_v_exception_message);
- __Pyx_GIVEREF(__pyx_v_exception_message);
- __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_14, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;}
- __Pyx_GOTREF(__pyx_t_4);
- __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0;
- __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
- __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;}
- __Pyx_GOTREF(__pyx_t_3);
- PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_4);
- __Pyx_GIVEREF(__pyx_t_4);
- __pyx_t_4 = 0;
- __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_IOError, __pyx_t_3, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;}
- __Pyx_GOTREF(__pyx_t_4);
- __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
-
- /* "kenlm.pyx":96
- * exception_message = str(exception).replace('\n', ' ')
- * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\
- * from exception # <<<<<<<<<<<<<<
- * self.vocab = &self.model.BaseVocabulary()
- *
- */
- __Pyx_Raise(__pyx_t_4, 0, 0, __pyx_v_exception);
- __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
- {__pyx_filename = __pyx_f[0]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;}
- }
- goto __pyx_L5_except_error;
- __pyx_L5_except_error:;
- __Pyx_XGIVEREF(__pyx_t_6);
- __Pyx_XGIVEREF(__pyx_t_7);
- __Pyx_XGIVEREF(__pyx_t_8);
- __Pyx_ExceptionReset(__pyx_t_6, __pyx_t_7, __pyx_t_8);
- goto __pyx_L1_error;
- __pyx_L10_try_end:;
- }
-
- /* "kenlm.pyx":97
- * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\
- * from exception
- * self.vocab = &self.model.BaseVocabulary() # <<<<<<<<<<<<<<
- *
- * def __dealloc__(self):
- */
- __pyx_v_self->vocab = (&__pyx_v_self->model->BaseVocabulary());
-
- /* "kenlm.pyx":84
- * cdef _kenlm.const_Vocabulary* vocab
- *
- * def __init__(self, path): # <<<<<<<<<<<<<<
- * """
- * Load the language model.
- */
-
- /* function exit code */
- __pyx_r = 0;
- goto __pyx_L0;
- __pyx_L1_error:;
- __Pyx_XDECREF(__pyx_t_1);
- __Pyx_XDECREF(__pyx_t_2);
- __Pyx_XDECREF(__pyx_t_3);
- __Pyx_XDECREF(__pyx_t_4);
- __Pyx_XDECREF(__pyx_t_5);
- __Pyx_XDECREF(__pyx_t_12);
- __Pyx_XDECREF(__pyx_t_14);
- __Pyx_AddTraceback("kenlm.LanguageModel.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename);
- __pyx_r = -1;
- __pyx_L0:;
- __Pyx_XDECREF(__pyx_v_exception);
- __Pyx_XDECREF(__pyx_v_exception_message);
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-/* "kenlm.pyx":99
- * self.vocab = &self.model.BaseVocabulary()
- *
- * def __dealloc__(self): # <<<<<<<<<<<<<<
- * del self.model
- *
- */
-
-/* Python wrapper */
-static void __pyx_pw_5kenlm_13LanguageModel_3__dealloc__(PyObject *__pyx_v_self); /*proto*/
-static void __pyx_pw_5kenlm_13LanguageModel_3__dealloc__(PyObject *__pyx_v_self) {
- __Pyx_RefNannyDeclarations
- __Pyx_RefNannySetupContext("__dealloc__ (wrapper)", 0);
- __pyx_pf_5kenlm_13LanguageModel_2__dealloc__(((struct __pyx_obj_5kenlm_LanguageModel *)__pyx_v_self));
-
- /* function exit code */
- __Pyx_RefNannyFinishContext();
-}
-
-static void __pyx_pf_5kenlm_13LanguageModel_2__dealloc__(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self) {
- __Pyx_RefNannyDeclarations
- __Pyx_RefNannySetupContext("__dealloc__", 0);
-
- /* "kenlm.pyx":100
- *
- * def __dealloc__(self):
- * del self.model # <<<<<<<<<<<<<<
- *
- * property order:
- */
- delete __pyx_v_self->model;
-
- /* "kenlm.pyx":99
- * self.vocab = &self.model.BaseVocabulary()
- *
- * def __dealloc__(self): # <<<<<<<<<<<<<<
- * del self.model
- *
- */
-
- /* function exit code */
- __Pyx_RefNannyFinishContext();
-}
-
-/* "kenlm.pyx":103
- *
- * property order:
- * def __get__(self): # <<<<<<<<<<<<<<
- * return self.model.Order()
- *
- */
-
-/* Python wrapper */
-static PyObject *__pyx_pw_5kenlm_13LanguageModel_5order_1__get__(PyObject *__pyx_v_self); /*proto*/
-static PyObject *__pyx_pw_5kenlm_13LanguageModel_5order_1__get__(PyObject *__pyx_v_self) {
- PyObject *__pyx_r = 0;
- __Pyx_RefNannyDeclarations
- __Pyx_RefNannySetupContext("__get__ (wrapper)", 0);
- __pyx_r = __pyx_pf_5kenlm_13LanguageModel_5order___get__(((struct __pyx_obj_5kenlm_LanguageModel *)__pyx_v_self));
-
- /* function exit code */
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-static PyObject *__pyx_pf_5kenlm_13LanguageModel_5order___get__(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self) {
- PyObject *__pyx_r = NULL;
- __Pyx_RefNannyDeclarations
- PyObject *__pyx_t_1 = NULL;
- int __pyx_lineno = 0;
- const char *__pyx_filename = NULL;
- int __pyx_clineno = 0;
- __Pyx_RefNannySetupContext("__get__", 0);
-
- /* "kenlm.pyx":104
- * property order:
- * def __get__(self):
- * return self.model.Order() # <<<<<<<<<<<<<<
- *
- * def score(self, sentence, bos = True, eos = True):
- */
- __Pyx_XDECREF(__pyx_r);
- __pyx_t_1 = __Pyx_PyInt_From_unsigned_int(__pyx_v_self->model->Order()); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 104; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_1);
- __pyx_r = __pyx_t_1;
- __pyx_t_1 = 0;
- goto __pyx_L0;
-
- /* "kenlm.pyx":103
- *
- * property order:
- * def __get__(self): # <<<<<<<<<<<<<<
- * return self.model.Order()
- *
- */
-
- /* function exit code */
- __pyx_L1_error:;
- __Pyx_XDECREF(__pyx_t_1);
- __Pyx_AddTraceback("kenlm.LanguageModel.order.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename);
- __pyx_r = NULL;
- __pyx_L0:;
- __Pyx_XGIVEREF(__pyx_r);
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-/* "kenlm.pyx":106
- * return self.model.Order()
- *
- * def score(self, sentence, bos = True, eos = True): # <<<<<<<<<<<<<<
- * cdef list words = as_str(sentence).split()
- * cdef _kenlm.State state
- */
-
-/* Python wrapper */
-static PyObject *__pyx_pw_5kenlm_13LanguageModel_5score(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
-static PyObject *__pyx_pw_5kenlm_13LanguageModel_5score(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
- PyObject *__pyx_v_sentence = 0;
- PyObject *__pyx_v_bos = 0;
- PyObject *__pyx_v_eos
<TRUNCATED>
[13/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/common/print.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/common/print.cc b/ext/kenlm/lm/common/print.cc
deleted file mode 100644
index 518b62f..0000000
--- a/ext/kenlm/lm/common/print.cc
+++ /dev/null
@@ -1,62 +0,0 @@
-#include "lm/common/print.hh"
-
-#include "lm/common/ngram_stream.hh"
-#include "util/file_stream.hh"
-#include "util/file.hh"
-#include "util/mmap.hh"
-#include "util/scoped.hh"
-
-#include <sstream>
-#include <cstring>
-
-namespace lm {
-
-VocabReconstitute::VocabReconstitute(int fd) {
- uint64_t size = util::SizeOrThrow(fd);
- util::MapRead(util::POPULATE_OR_READ, fd, 0, size, memory_);
- const char *const start = static_cast<const char*>(memory_.get());
- const char *i;
- for (i = start; i != start + size; i += strlen(i) + 1) {
- map_.push_back(i);
- }
- // Last one for LookupPiece.
- map_.push_back(i);
-}
-
-namespace {
-template <class Payload> void PrintLead(const VocabReconstitute &vocab, ProxyStream<Payload> &stream, util::FileStream &out) {
- out << stream->Value().prob << '\t' << vocab.Lookup(*stream->begin());
- for (const WordIndex *i = stream->begin() + 1; i != stream->end(); ++i) {
- out << ' ' << vocab.Lookup(*i);
- }
-}
-} // namespace
-
-void PrintARPA::Run(const util::stream::ChainPositions &positions) {
- VocabReconstitute vocab(vocab_fd_);
- util::FileStream out(out_fd_);
- out << "\\data\\\n";
- for (size_t i = 0; i < positions.size(); ++i) {
- out << "ngram " << (i+1) << '=' << counts_[i] << '\n';
- }
- out << '\n';
-
- for (unsigned order = 1; order < positions.size(); ++order) {
- out << "\\" << order << "-grams:" << '\n';
- for (ProxyStream<NGram<ProbBackoff> > stream(positions[order - 1], NGram<ProbBackoff>(NULL, order)); stream; ++stream) {
- PrintLead(vocab, stream, out);
- out << '\t' << stream->Value().backoff << '\n';
- }
- out << '\n';
- }
-
- out << "\\" << positions.size() << "-grams:" << '\n';
- for (ProxyStream<NGram<Prob> > stream(positions.back(), NGram<Prob>(NULL, positions.size())); stream; ++stream) {
- PrintLead(vocab, stream, out);
- out << '\n';
- }
- out << '\n';
- out << "\\end\\\n";
-}
-
-} // namespace lm
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/common/print.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/common/print.hh b/ext/kenlm/lm/common/print.hh
deleted file mode 100644
index 6aa08b3..0000000
--- a/ext/kenlm/lm/common/print.hh
+++ /dev/null
@@ -1,58 +0,0 @@
-#ifndef LM_COMMON_PRINT_H
-#define LM_COMMON_PRINT_H
-
-#include "lm/word_index.hh"
-#include "util/mmap.hh"
-#include "util/string_piece.hh"
-
-#include <cassert>
-#include <vector>
-
-namespace util { namespace stream { class ChainPositions; }}
-
-// Warning: PrintARPA routines read all unigrams before all bigrams before all
-// trigrams etc. So if other parts of the chain move jointly, you'll have to
-// buffer.
-
-namespace lm {
-
-class VocabReconstitute {
- public:
- // fd must be alive for life of this object; does not take ownership.
- explicit VocabReconstitute(int fd);
-
- const char *Lookup(WordIndex index) const {
- assert(index < map_.size() - 1);
- return map_[index];
- }
-
- StringPiece LookupPiece(WordIndex index) const {
- return StringPiece(map_[index], map_[index + 1] - 1 - map_[index]);
- }
-
- std::size_t Size() const {
- // There's an extra entry to support StringPiece lengths.
- return map_.size() - 1;
- }
-
- private:
- util::scoped_memory memory_;
- std::vector<const char*> map_;
-};
-
-class PrintARPA {
- public:
- // Does not take ownership of vocab_fd or out_fd.
- explicit PrintARPA(int vocab_fd, int out_fd, const std::vector<uint64_t> &counts)
- : vocab_fd_(vocab_fd), out_fd_(out_fd), counts_(counts) {}
-
- void Run(const util::stream::ChainPositions &positions);
-
- private:
- int vocab_fd_;
- int out_fd_;
- std::vector<uint64_t> counts_;
-};
-
-} // namespace lm
-#endif // LM_COMMON_PRINT_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/common/renumber.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/common/renumber.cc b/ext/kenlm/lm/common/renumber.cc
deleted file mode 100644
index 0632a14..0000000
--- a/ext/kenlm/lm/common/renumber.cc
+++ /dev/null
@@ -1,17 +0,0 @@
-#include "lm/common/renumber.hh"
-#include "lm/common/ngram.hh"
-
-#include "util/stream/stream.hh"
-
-namespace lm {
-
-void Renumber::Run(const util::stream::ChainPosition &position) {
- for (util::stream::Stream stream(position); stream; ++stream) {
- NGramHeader gram(stream.Get(), order_);
- for (WordIndex *w = gram.begin(); w != gram.end(); ++w) {
- *w = new_numbers_[*w];
- }
- }
-}
-
-} // namespace lm
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/common/renumber.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/common/renumber.hh b/ext/kenlm/lm/common/renumber.hh
deleted file mode 100644
index ca25c4d..0000000
--- a/ext/kenlm/lm/common/renumber.hh
+++ /dev/null
@@ -1,30 +0,0 @@
-/* Map vocab ids. This is useful to merge independently collected counts or
- * change the vocab ids to the order used by the trie.
- */
-#ifndef LM_COMMON_RENUMBER_H
-#define LM_COMMON_RENUMBER_H
-
-#include "lm/word_index.hh"
-
-#include <cstddef>
-
-namespace util { namespace stream { class ChainPosition; }}
-
-namespace lm {
-
-class Renumber {
- public:
- // Assumes the array is large enough to map all words and stays alive while
- // the thread is active.
- Renumber(const WordIndex *new_numbers, std::size_t order)
- : new_numbers_(new_numbers), order_(order) {}
-
- void Run(const util::stream::ChainPosition &position);
-
- private:
- const WordIndex *new_numbers_;
- std::size_t order_;
-};
-
-} // namespace lm
-#endif // LM_COMMON_RENUMBER_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/common/size_option.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/common/size_option.cc b/ext/kenlm/lm/common/size_option.cc
deleted file mode 100644
index 46a920e..0000000
--- a/ext/kenlm/lm/common/size_option.cc
+++ /dev/null
@@ -1,24 +0,0 @@
-#include <boost/program_options.hpp>
-#include "util/usage.hh"
-
-namespace lm {
-
-namespace {
-class SizeNotify {
- public:
- explicit SizeNotify(std::size_t &out) : behind_(out) {}
-
- void operator()(const std::string &from) {
- behind_ = util::ParseSize(from);
- }
-
- private:
- std::size_t &behind_;
-};
-}
-
-boost::program_options::typed_value<std::string> *SizeOption(std::size_t &to, const char *default_value) {
- return boost::program_options::value<std::string>()->notifier(SizeNotify(to))->default_value(default_value);
-}
-
-} // namespace lm
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/common/size_option.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/common/size_option.hh b/ext/kenlm/lm/common/size_option.hh
deleted file mode 100644
index d3b8e33..0000000
--- a/ext/kenlm/lm/common/size_option.hh
+++ /dev/null
@@ -1,11 +0,0 @@
-#include <boost/program_options.hpp>
-
-#include <cstddef>
-#include <string>
-
-namespace lm {
-
-// Create a boost program option for data sizes. This parses sizes like 1T and 10k.
-boost::program_options::typed_value<std::string> *SizeOption(std::size_t &to, const char *default_value);
-
-} // namespace lm
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/common/special.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/common/special.hh b/ext/kenlm/lm/common/special.hh
deleted file mode 100644
index 0677cd7..0000000
--- a/ext/kenlm/lm/common/special.hh
+++ /dev/null
@@ -1,27 +0,0 @@
-#ifndef LM_COMMON_SPECIAL_H
-#define LM_COMMON_SPECIAL_H
-
-#include "lm/word_index.hh"
-
-namespace lm {
-
-class SpecialVocab {
- public:
- SpecialVocab(WordIndex bos, WordIndex eos) : bos_(bos), eos_(eos) {}
-
- bool IsSpecial(WordIndex word) const {
- return word == kUNK || word == bos_ || word == eos_;
- }
-
- WordIndex UNK() const { return kUNK; }
- WordIndex BOS() const { return bos_; }
- WordIndex EOS() const { return eos_; }
-
- private:
- WordIndex bos_;
- WordIndex eos_;
-};
-
-} // namespace lm
-
-#endif // LM_COMMON_SPECIAL_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/config.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/config.cc b/ext/kenlm/lm/config.cc
deleted file mode 100644
index 6c695ed..0000000
--- a/ext/kenlm/lm/config.cc
+++ /dev/null
@@ -1,30 +0,0 @@
-#include "lm/config.hh"
-
-#include <iostream>
-
-namespace lm {
-namespace ngram {
-
-Config::Config() :
- show_progress(true),
- messages(&std::cerr),
- enumerate_vocab(NULL),
- unknown_missing(COMPLAIN),
- sentence_marker_missing(THROW_UP),
- positive_log_probability(THROW_UP),
- unknown_missing_logprob(-100.0),
- probing_multiplier(1.5),
- building_memory(1073741824ULL), // 1 GB
- temporary_directory_prefix(""),
- arpa_complain(ALL),
- write_mmap(NULL),
- write_method(WRITE_AFTER),
- include_vocab(true),
- rest_function(REST_MAX),
- prob_bits(8),
- backoff_bits(8),
- pointer_bhiksha_bits(22),
- load_method(util::POPULATE_OR_READ) {}
-
-} // namespace ngram
-} // namespace lm
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/config.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/config.hh b/ext/kenlm/lm/config.hh
deleted file mode 100644
index 21b9e7e..0000000
--- a/ext/kenlm/lm/config.hh
+++ /dev/null
@@ -1,124 +0,0 @@
-#ifndef LM_CONFIG_H
-#define LM_CONFIG_H
-
-#include "lm/lm_exception.hh"
-#include "util/mmap.hh"
-
-#include <iosfwd>
-#include <string>
-#include <vector>
-
-/* Configuration for ngram model. Separate header to reduce pollution. */
-
-namespace lm {
-
-class EnumerateVocab;
-
-namespace ngram {
-
-struct Config {
- // EFFECTIVE FOR BOTH ARPA AND BINARY READS
-
- // (default true) print progress bar to messages
- bool show_progress;
-
- // Where to log messages including the progress bar. Set to NULL for
- // silence.
- std::ostream *messages;
-
- std::ostream *ProgressMessages() const {
- return show_progress ? messages : 0;
- }
-
- // This will be called with every string in the vocabulary by the
- // constructor; it need only exist for the lifetime of the constructor.
- // See enumerate_vocab.hh for more detail. Config does not take ownership;
- // just delete/let it go out of scope after the constructor exits.
- EnumerateVocab *enumerate_vocab;
-
-
- // ONLY EFFECTIVE WHEN READING ARPA
-
- // What to do when <unk> isn't in the provided model.
- WarningAction unknown_missing;
- // What to do when <s> or </s> is missing from the model.
- // If THROW_UP, the exception will be of type util::SpecialWordMissingException.
- WarningAction sentence_marker_missing;
-
- // What to do with a positive log probability. For COMPLAIN and SILENT, map
- // to 0.
- WarningAction positive_log_probability;
-
- // The probability to substitute for <unk> if it's missing from the model.
- // No effect if the model has <unk> or unknown_missing == THROW_UP.
- float unknown_missing_logprob;
-
- // Size multiplier for probing hash table. Must be > 1. Space is linear in
- // this. Time is probing_multiplier / (probing_multiplier - 1). No effect
- // for sorted variant.
- // If you find yourself setting this to a low number, consider using the
- // TrieModel which has lower memory consumption.
- float probing_multiplier;
-
- // Amount of memory to use for building. The actual memory usage will be
- // higher since this just sets sort buffer size. Only applies to trie
- // models.
- std::size_t building_memory;
-
- // Template for temporary directory appropriate for passing to mkdtemp.
- // The characters XXXXXX are appended before passing to mkdtemp. Only
- // applies to trie. If empty, defaults to write_mmap. If that's NULL,
- // defaults to input file name.
- std::string temporary_directory_prefix;
-
- // Level of complaining to do when loading from ARPA instead of binary format.
- enum ARPALoadComplain {ALL, EXPENSIVE, NONE};
- ARPALoadComplain arpa_complain;
-
- // While loading an ARPA file, also write out this binary format file. Set
- // to NULL to disable.
- const char *write_mmap;
-
- enum WriteMethod {
- WRITE_MMAP, // Map the file directly.
- WRITE_AFTER // Write after we're done.
- };
- WriteMethod write_method;
-
- // Include the vocab in the binary file? Only effective if write_mmap != NULL.
- bool include_vocab;
-
-
- // Left rest options. Only used when the model includes rest costs.
- enum RestFunction {
- REST_MAX, // Maximum of any score to the left
- REST_LOWER, // Use lower-order files given below.
- };
- RestFunction rest_function;
- // Only used for REST_LOWER.
- std::vector<std::string> rest_lower_files;
-
-
- // Quantization options. Only effective for QuantTrieModel. One value is
- // reserved for each of prob and backoff, so 2^bits - 1 buckets will be used
- // to quantize (and one of the remaining backoffs will be 0).
- uint8_t prob_bits, backoff_bits;
-
- // Bhiksha compression (simple form). Only works with trie.
- uint8_t pointer_bhiksha_bits;
-
-
- // ONLY EFFECTIVE WHEN READING BINARY
-
- // How to get the giant array into memory: lazy mmap, populate, read etc.
- // See util/mmap.hh for details of MapMethod.
- util::LoadMethod load_method;
-
-
- // Set defaults.
- Config();
-};
-
-} /* namespace ngram */ } /* namespace lm */
-
-#endif // LM_CONFIG_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/enumerate_vocab.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/enumerate_vocab.hh b/ext/kenlm/lm/enumerate_vocab.hh
deleted file mode 100644
index f4c94cd..0000000
--- a/ext/kenlm/lm/enumerate_vocab.hh
+++ /dev/null
@@ -1,28 +0,0 @@
-#ifndef LM_ENUMERATE_VOCAB_H
-#define LM_ENUMERATE_VOCAB_H
-
-#include "lm/word_index.hh"
-#include "util/string_piece.hh"
-
-namespace lm {
-
-/* If you need the actual strings in the vocabulary, inherit from this class
- * and implement Add. Then put a pointer in Config.enumerate_vocab; it does
- * not take ownership. Add is called once per vocab word. index starts at 0
- * and increases by 1 each time. This is only used by the Model constructor;
- * the pointer is not retained by the class.
- */
-class EnumerateVocab {
- public:
- virtual ~EnumerateVocab() {}
-
- virtual void Add(WordIndex index, const StringPiece &str) = 0;
-
- protected:
- EnumerateVocab() {}
-};
-
-} // namespace lm
-
-#endif // LM_ENUMERATE_VOCAB_H
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/facade.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/facade.hh b/ext/kenlm/lm/facade.hh
deleted file mode 100644
index 325ef15..0000000
--- a/ext/kenlm/lm/facade.hh
+++ /dev/null
@@ -1,73 +0,0 @@
-#ifndef LM_FACADE_H
-#define LM_FACADE_H
-
-#include "lm/virtual_interface.hh"
-#include "util/string_piece.hh"
-
-#include <string>
-
-namespace lm {
-namespace base {
-
-// Common model interface that depends on knowing the specific classes.
-// Curiously recurring template pattern.
-template <class Child, class StateT, class VocabularyT> class ModelFacade : public Model {
- public:
- typedef StateT State;
- typedef VocabularyT Vocabulary;
-
- /* Translate from void* to State */
- FullScoreReturn BaseFullScore(const void *in_state, const WordIndex new_word, void *out_state) const {
- return static_cast<const Child*>(this)->FullScore(
- *reinterpret_cast<const State*>(in_state),
- new_word,
- *reinterpret_cast<State*>(out_state));
- }
-
- FullScoreReturn BaseFullScoreForgotState(const WordIndex *context_rbegin, const WordIndex *context_rend, const WordIndex new_word, void *out_state) const {
- return static_cast<const Child*>(this)->FullScoreForgotState(
- context_rbegin,
- context_rend,
- new_word,
- *reinterpret_cast<State*>(out_state));
- }
-
- // Default Score function calls FullScore. Model can override this.
- float Score(const State &in_state, const WordIndex new_word, State &out_state) const {
- return static_cast<const Child*>(this)->FullScore(in_state, new_word, out_state).prob;
- }
-
- float BaseScore(const void *in_state, const WordIndex new_word, void *out_state) const {
- return static_cast<const Child*>(this)->Score(
- *reinterpret_cast<const State*>(in_state),
- new_word,
- *reinterpret_cast<State*>(out_state));
- }
-
- const State &BeginSentenceState() const { return begin_sentence_; }
- const State &NullContextState() const { return null_context_; }
- const Vocabulary &GetVocabulary() const { return *static_cast<const Vocabulary*>(&BaseVocabulary()); }
-
- protected:
- ModelFacade() : Model(sizeof(State)) {}
-
- virtual ~ModelFacade() {}
-
- // begin_sentence and null_context can disappear after. vocab should stay.
- void Init(const State &begin_sentence, const State &null_context, const Vocabulary &vocab, unsigned char order) {
- begin_sentence_ = begin_sentence;
- null_context_ = null_context;
- begin_sentence_memory_ = &begin_sentence_;
- null_context_memory_ = &null_context_;
- base_vocab_ = &vocab;
- order_ = order;
- }
-
- private:
- State begin_sentence_, null_context_;
-};
-
-} // mamespace base
-} // namespace lm
-
-#endif // LM_FACADE_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/filter/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/filter/CMakeLists.txt b/ext/kenlm/lm/filter/CMakeLists.txt
deleted file mode 100644
index d4616cc..0000000
--- a/ext/kenlm/lm/filter/CMakeLists.txt
+++ /dev/null
@@ -1,62 +0,0 @@
-cmake_minimum_required(VERSION 2.8.8)
-#
-# The KenLM cmake files make use of add_library(... OBJECTS ...)
-#
-# This syntax allows grouping of source files when compiling
-# (effectively creating "fake" libraries based on source subdirs).
-#
-# This syntax was only added in cmake version 2.8.8
-#
-# see http://www.cmake.org/Wiki/CMake/Tutorials/Object_Library
-
-
-# This CMake file was created by Lane Schwartz <do...@gmail.com>
-
-# Explicitly list the source files for this subdirectory
-#
-# If you add any source files to this subdirectory
-# that should be included in the kenlm library,
-# (this excludes any unit test files)
-# you should add them to the following list:
-#
-# In order to set correct paths to these files
-# in case this variable is referenced by CMake files in the parent directory,
-# we prefix all files with ${CMAKE_CURRENT_SOURCE_DIR}.
-#
-set(KENLM_FILTER_SOURCE
- ${CMAKE_CURRENT_SOURCE_DIR}/arpa_io.cc
- ${CMAKE_CURRENT_SOURCE_DIR}/phrase.cc
- ${CMAKE_CURRENT_SOURCE_DIR}/vocab.cc
- )
-
-
-# Group these objects together for later use.
-#
-# Given add_library(foo OBJECT ${my_foo_sources}),
-# refer to these objects as $<TARGET_OBJECTS:foo>
-#
-add_library(kenlm_filter OBJECT ${KENLM_FILTER_SOURCE})
-
-
-# Explicitly list the executable files to be compiled
-set(EXE_LIST
- filter
- phrase_table_vocab
-)
-
-
-# Iterate through the executable list
-foreach(exe ${EXE_LIST})
-
- # Compile the executable, linking against the requisite dependent object files
- add_executable(${exe} ${exe}_main.cc $<TARGET_OBJECTS:kenlm> $<TARGET_OBJECTS:kenlm_filter> $<TARGET_OBJECTS:kenlm_util>)
-
- # Link the executable against boost
- target_link_libraries(${exe} ${Boost_LIBRARIES} pthread)
-
- # Group executables together
- set_target_properties(${exe} PROPERTIES FOLDER executables)
-
-# End for loop
-endforeach(exe)
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/filter/Jamfile
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/filter/Jamfile b/ext/kenlm/lm/filter/Jamfile
deleted file mode 100644
index bcf62da..0000000
--- a/ext/kenlm/lm/filter/Jamfile
+++ /dev/null
@@ -1,7 +0,0 @@
-fakelib lm_filter : phrase.cc vocab.cc arpa_io.cc ../../util//kenutil : <threading>multi:<library>/top//boost_thread ;
-
-obj main : filter_main.cc : <threading>single:<define>NTHREAD <include>../.. ;
-
-exe filter : main lm_filter ../../util//kenutil ..//kenlm : <threading>multi:<library>/top//boost_thread ;
-
-exe phrase_table_vocab : phrase_table_vocab_main.cc ../../util//kenutil ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/filter/arpa_io.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/filter/arpa_io.cc b/ext/kenlm/lm/filter/arpa_io.cc
deleted file mode 100644
index 2cae60f..0000000
--- a/ext/kenlm/lm/filter/arpa_io.cc
+++ /dev/null
@@ -1,77 +0,0 @@
-#include "lm/filter/arpa_io.hh"
-#include "util/file_piece.hh"
-#include "util/string_stream.hh"
-
-#include <iostream>
-#include <ostream>
-#include <string>
-#include <vector>
-
-#include <cctype>
-#include <cerrno>
-#include <cstring>
-
-namespace lm {
-
-ARPAInputException::ARPAInputException(const StringPiece &message) throw() {
- *this << message;
-}
-
-ARPAInputException::ARPAInputException(const StringPiece &message, const StringPiece &line) throw() {
- *this << message << " in line " << line;
-}
-
-ARPAInputException::~ARPAInputException() throw() {}
-
-// Seeking is the responsibility of the caller.
-template <class Stream> void WriteCounts(Stream &out, const std::vector<uint64_t> &number) {
- out << "\n\\data\\\n";
- for (unsigned int i = 0; i < number.size(); ++i) {
- out << "ngram " << i+1 << "=" << number[i] << '\n';
- }
- out << '\n';
-}
-
-size_t SizeNeededForCounts(const std::vector<uint64_t> &number) {
- std::string buf;
- util::StringStream stream(buf);
- WriteCounts(stream, number);
- return buf.size();
-}
-
-bool IsEntirelyWhiteSpace(const StringPiece &line) {
- for (size_t i = 0; i < static_cast<size_t>(line.size()); ++i) {
- if (!isspace(line.data()[i])) return false;
- }
- return true;
-}
-
-ARPAOutput::ARPAOutput(const char *name, size_t buffer_size)
- : file_backing_(util::CreateOrThrow(name)), file_(file_backing_.get(), buffer_size) {}
-
-void ARPAOutput::ReserveForCounts(std::streampos reserve) {
- for (std::streampos i = 0; i < reserve; i += std::streampos(1)) {
- file_ << '\n';
- }
-}
-
-void ARPAOutput::BeginLength(unsigned int length) {
- file_ << '\\' << length << "-grams:" << '\n';
-}
-
-void ARPAOutput::EndLength(unsigned int length) {
- file_ << '\n';
- if (length > counts_.size()) {
- counts_.resize(length);
- }
- counts_[length - 1] = fast_counter_;
-}
-
-void ARPAOutput::Finish() {
- file_ << "\\end\\\n";
- file_.seekp(0);
- WriteCounts(file_, counts_);
- file_.flush();
-}
-
-} // namespace lm
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/filter/arpa_io.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/filter/arpa_io.hh b/ext/kenlm/lm/filter/arpa_io.hh
deleted file mode 100644
index 7489270..0000000
--- a/ext/kenlm/lm/filter/arpa_io.hh
+++ /dev/null
@@ -1,99 +0,0 @@
-#ifndef LM_FILTER_ARPA_IO_H
-#define LM_FILTER_ARPA_IO_H
-/* Input and output for ARPA format language model files.
- */
-#include "lm/read_arpa.hh"
-#include "util/exception.hh"
-#include "util/file_stream.hh"
-#include "util/string_piece.hh"
-#include "util/tokenize_piece.hh"
-
-#include <boost/noncopyable.hpp>
-#include <boost/scoped_array.hpp>
-
-#include <fstream>
-#include <string>
-#include <vector>
-
-#include <cstring>
-#include <stdint.h>
-
-namespace util { class FilePiece; }
-
-namespace lm {
-
-class ARPAInputException : public util::Exception {
- public:
- explicit ARPAInputException(const StringPiece &message) throw();
- explicit ARPAInputException(const StringPiece &message, const StringPiece &line) throw();
- virtual ~ARPAInputException() throw();
-};
-
-// Handling for the counts of n-grams at the beginning of ARPA files.
-size_t SizeNeededForCounts(const std::vector<uint64_t> &number);
-
-/* Writes an ARPA file. This has to be seekable so the counts can be written
- * at the end. Hence, I just have it own a std::fstream instead of accepting
- * a separately held std::ostream. TODO: use the fast one from estimation.
- */
-class ARPAOutput : boost::noncopyable {
- public:
- explicit ARPAOutput(const char *name, size_t buffer_size = 65536);
-
- void ReserveForCounts(std::streampos reserve);
-
- void BeginLength(unsigned int length);
-
- void AddNGram(const StringPiece &line) {
- file_ << line << '\n';
- ++fast_counter_;
- }
-
- void AddNGram(const StringPiece &ngram, const StringPiece &line) {
- AddNGram(line);
- }
-
- template <class Iterator> void AddNGram(const Iterator &begin, const Iterator &end, const StringPiece &line) {
- AddNGram(line);
- }
-
- void EndLength(unsigned int length);
-
- void Finish();
-
- private:
- util::scoped_fd file_backing_;
- util::FileStream file_;
- size_t fast_counter_;
- std::vector<uint64_t> counts_;
-};
-
-
-template <class Output> void ReadNGrams(util::FilePiece &in, unsigned int length, uint64_t number, Output &out) {
- ReadNGramHeader(in, length);
- out.BeginLength(length);
- for (uint64_t i = 0; i < number; ++i) {
- StringPiece line = in.ReadLine();
- util::TokenIter<util::SingleCharacter> tabber(line, '\t');
- if (!tabber) throw ARPAInputException("blank line", line);
- if (!++tabber) throw ARPAInputException("no tab", line);
-
- out.AddNGram(*tabber, line);
- }
- out.EndLength(length);
-}
-
-template <class Output> void ReadARPA(util::FilePiece &in_lm, Output &out) {
- std::vector<uint64_t> number;
- ReadARPACounts(in_lm, number);
- out.ReserveForCounts(SizeNeededForCounts(number));
- for (unsigned int i = 0; i < number.size(); ++i) {
- ReadNGrams(in_lm, i + 1, number[i], out);
- }
- ReadEnd(in_lm);
- out.Finish();
-}
-
-} // namespace lm
-
-#endif // LM_FILTER_ARPA_IO_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/filter/count_io.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/filter/count_io.hh b/ext/kenlm/lm/filter/count_io.hh
deleted file mode 100644
index 1af6676..0000000
--- a/ext/kenlm/lm/filter/count_io.hh
+++ /dev/null
@@ -1,89 +0,0 @@
-#ifndef LM_FILTER_COUNT_IO_H
-#define LM_FILTER_COUNT_IO_H
-
-#include <fstream>
-#include <iostream>
-#include <string>
-
-#include "util/file_stream.hh"
-#include "util/file.hh"
-#include "util/file_piece.hh"
-
-namespace lm {
-
-class CountOutput : boost::noncopyable {
- public:
- explicit CountOutput(const char *name) : file_(util::CreateOrThrow(name)) {}
-
- void AddNGram(const StringPiece &line) {
- file_ << line << '\n';
- }
-
- template <class Iterator> void AddNGram(const Iterator &begin, const Iterator &end, const StringPiece &line) {
- AddNGram(line);
- }
-
- void AddNGram(const StringPiece &ngram, const StringPiece &line) {
- AddNGram(line);
- }
-
- private:
- util::FileStream file_;
-};
-
-class CountBatch {
- public:
- explicit CountBatch(std::streamsize initial_read)
- : initial_read_(initial_read) {
- buffer_.reserve(initial_read);
- }
-
- void Read(std::istream &in) {
- buffer_.resize(initial_read_);
- in.read(&*buffer_.begin(), initial_read_);
- buffer_.resize(in.gcount());
- char got;
- while (in.get(got) && got != '\n')
- buffer_.push_back(got);
- }
-
- template <class Output> void Send(Output &out) {
- for (util::TokenIter<util::SingleCharacter> line(StringPiece(&*buffer_.begin(), buffer_.size()), '\n'); line; ++line) {
- util::TokenIter<util::SingleCharacter> tabber(*line, '\t');
- if (!tabber) {
- std::cerr << "Warning: empty n-gram count line being removed\n";
- continue;
- }
- util::TokenIter<util::SingleCharacter, true> words(*tabber, ' ');
- if (!words) {
- std::cerr << "Line has a tab but no words.\n";
- continue;
- }
- out.AddNGram(words, util::TokenIter<util::SingleCharacter, true>::end(), *line);
- }
- }
-
- private:
- std::streamsize initial_read_;
-
- // This could have been a std::string but that's less happy with raw writes.
- std::vector<char> buffer_;
-};
-
-template <class Output> void ReadCount(util::FilePiece &in_file, Output &out) {
- try {
- while (true) {
- StringPiece line = in_file.ReadLine();
- util::TokenIter<util::SingleCharacter> tabber(line, '\t');
- if (!tabber) {
- std::cerr << "Warning: empty n-gram count line being removed\n";
- continue;
- }
- out.AddNGram(*tabber, line);
- }
- } catch (const util::EndOfFileException &e) {}
-}
-
-} // namespace lm
-
-#endif // LM_FILTER_COUNT_IO_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/filter/filter_main.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/filter/filter_main.cc b/ext/kenlm/lm/filter/filter_main.cc
deleted file mode 100644
index 6e89d1f..0000000
--- a/ext/kenlm/lm/filter/filter_main.cc
+++ /dev/null
@@ -1,253 +0,0 @@
-#include "lm/filter/arpa_io.hh"
-#include "lm/filter/format.hh"
-#include "lm/filter/phrase.hh"
-#ifndef NTHREAD
-#include "lm/filter/thread.hh"
-#endif
-#include "lm/filter/vocab.hh"
-#include "lm/filter/wrapper.hh"
-#include "util/exception.hh"
-#include "util/file_piece.hh"
-
-#include <boost/ptr_container/ptr_vector.hpp>
-
-#include <cstring>
-#include <fstream>
-#include <iostream>
-#include <memory>
-
-namespace lm {
-namespace {
-
-void DisplayHelp(const char *name) {
- std::cerr
- << "Usage: " << name << " mode [context] [phrase] [raw|arpa] [threads:m] [batch_size:m] (vocab|model):input_file output_file\n\n"
- "copy mode just copies, but makes the format nicer for e.g. irstlm's broken\n"
- " parser.\n"
- "single mode treats the entire input as a single sentence.\n"
- "multiple mode filters to multiple sentences in parallel. Each sentence is on\n"
- " a separate line. A separate file is created for each sentence by appending\n"
- " the 0-indexed line number to the output file name.\n"
- "union mode produces one filtered model that is the union of models created by\n"
- " multiple mode.\n\n"
- "context means only the context (all but last word) has to pass the filter, but\n"
- " the entire n-gram is output.\n\n"
- "phrase means that the vocabulary is actually tab-delimited phrases and that the\n"
- " phrases can generate the n-gram when assembled in arbitrary order and\n"
- " clipped. Currently works with multiple or union mode.\n\n"
- "The file format is set by [raw|arpa] with default arpa:\n"
- "raw means space-separated tokens, optionally followed by a tab and arbitrary\n"
- " text. This is useful for ngram count files.\n"
- "arpa means the ARPA file format for n-gram language models.\n\n"
-#ifndef NTHREAD
- "threads:m sets m threads (default: conccurrency detected by boost)\n"
- "batch_size:m sets the batch size for threading. Expect memory usage from this\n"
- " of 2*threads*batch_size n-grams.\n\n"
-#else
- "This binary was compiled with -DNTHREAD, disabling threading. If you wanted\n"
- " threading, compile without this flag against Boost >=1.42.0.\n\n"
-#endif
- "There are two inputs: vocabulary and model. Either may be given as a file\n"
- " while the other is on stdin. Specify the type given as a file using\n"
- " vocab: or model: before the file name. \n\n"
- "For ARPA format, the output must be seekable. For raw format, it can be a\n"
- " stream i.e. /dev/stdout\n";
-}
-
-typedef enum {MODE_COPY, MODE_SINGLE, MODE_MULTIPLE, MODE_UNION, MODE_UNSET} FilterMode;
-typedef enum {FORMAT_ARPA, FORMAT_COUNT} Format;
-
-struct Config {
- Config() :
-#ifndef NTHREAD
- batch_size(25000),
- threads(boost::thread::hardware_concurrency()),
-#endif
- phrase(false),
- context(false),
- format(FORMAT_ARPA)
- {
-#ifndef NTHREAD
- if (!threads) threads = 1;
-#endif
- }
-
-#ifndef NTHREAD
- size_t batch_size;
- size_t threads;
-#endif
- bool phrase;
- bool context;
- FilterMode mode;
- Format format;
-};
-
-template <class Format, class Filter, class OutputBuffer, class Output> void RunThreadedFilter(const Config &config, util::FilePiece &in_lm, Filter &filter, Output &output) {
-#ifndef NTHREAD
- if (config.threads == 1) {
-#endif
- Format::RunFilter(in_lm, filter, output);
-#ifndef NTHREAD
- } else {
- typedef Controller<Filter, OutputBuffer, Output> Threaded;
- Threaded threading(config.batch_size, config.threads * 2, config.threads, filter, output);
- Format::RunFilter(in_lm, threading, output);
- }
-#endif
-}
-
-template <class Format, class Filter, class OutputBuffer, class Output> void RunContextFilter(const Config &config, util::FilePiece &in_lm, Filter filter, Output &output) {
- if (config.context) {
- ContextFilter<Filter> context_filter(filter);
- RunThreadedFilter<Format, ContextFilter<Filter>, OutputBuffer, Output>(config, in_lm, context_filter, output);
- } else {
- RunThreadedFilter<Format, Filter, OutputBuffer, Output>(config, in_lm, filter, output);
- }
-}
-
-template <class Format, class Binary> void DispatchBinaryFilter(const Config &config, util::FilePiece &in_lm, const Binary &binary, typename Format::Output &out) {
- typedef BinaryFilter<Binary> Filter;
- RunContextFilter<Format, Filter, BinaryOutputBuffer, typename Format::Output>(config, in_lm, Filter(binary), out);
-}
-
-template <class Format> void DispatchFilterModes(const Config &config, std::istream &in_vocab, util::FilePiece &in_lm, const char *out_name) {
- if (config.mode == MODE_MULTIPLE) {
- if (config.phrase) {
- typedef phrase::Multiple Filter;
- phrase::Substrings substrings;
- typename Format::Multiple out(out_name, phrase::ReadMultiple(in_vocab, substrings));
- RunContextFilter<Format, Filter, MultipleOutputBuffer, typename Format::Multiple>(config, in_lm, Filter(substrings), out);
- } else {
- typedef vocab::Multiple Filter;
- boost::unordered_map<std::string, std::vector<unsigned int> > words;
- typename Format::Multiple out(out_name, vocab::ReadMultiple(in_vocab, words));
- RunContextFilter<Format, Filter, MultipleOutputBuffer, typename Format::Multiple>(config, in_lm, Filter(words), out);
- }
- return;
- }
-
- typename Format::Output out(out_name);
-
- if (config.mode == MODE_COPY) {
- Format::Copy(in_lm, out);
- return;
- }
-
- if (config.mode == MODE_SINGLE) {
- vocab::Single::Words words;
- vocab::ReadSingle(in_vocab, words);
- DispatchBinaryFilter<Format, vocab::Single>(config, in_lm, vocab::Single(words), out);
- return;
- }
-
- if (config.mode == MODE_UNION) {
- if (config.phrase) {
- phrase::Substrings substrings;
- phrase::ReadMultiple(in_vocab, substrings);
- DispatchBinaryFilter<Format, phrase::Union>(config, in_lm, phrase::Union(substrings), out);
- } else {
- vocab::Union::Words words;
- vocab::ReadMultiple(in_vocab, words);
- DispatchBinaryFilter<Format, vocab::Union>(config, in_lm, vocab::Union(words), out);
- }
- return;
- }
-}
-
-} // namespace
-} // namespace lm
-
-int main(int argc, char *argv[]) {
- try {
- if (argc < 4) {
- lm::DisplayHelp(argv[0]);
- return 1;
- }
-
- // I used to have boost::program_options, but some users didn't want to compile boost.
- lm::Config config;
- config.mode = lm::MODE_UNSET;
- for (int i = 1; i < argc - 2; ++i) {
- const char *str = argv[i];
- if (!std::strcmp(str, "copy")) {
- config.mode = lm::MODE_COPY;
- } else if (!std::strcmp(str, "single")) {
- config.mode = lm::MODE_SINGLE;
- } else if (!std::strcmp(str, "multiple")) {
- config.mode = lm::MODE_MULTIPLE;
- } else if (!std::strcmp(str, "union")) {
- config.mode = lm::MODE_UNION;
- } else if (!std::strcmp(str, "phrase")) {
- config.phrase = true;
- } else if (!std::strcmp(str, "context")) {
- config.context = true;
- } else if (!std::strcmp(str, "arpa")) {
- config.format = lm::FORMAT_ARPA;
- } else if (!std::strcmp(str, "raw")) {
- config.format = lm::FORMAT_COUNT;
-#ifndef NTHREAD
- } else if (!std::strncmp(str, "threads:", 8)) {
- config.threads = boost::lexical_cast<size_t>(str + 8);
- if (!config.threads) {
- std::cerr << "Specify at least one thread." << std::endl;
- return 1;
- }
- } else if (!std::strncmp(str, "batch_size:", 11)) {
- config.batch_size = boost::lexical_cast<size_t>(str + 11);
- if (config.batch_size < 5000) {
- std::cerr << "Batch size must be at least one and should probably be >= 5000" << std::endl;
- if (!config.batch_size) return 1;
- }
-#endif
- } else {
- lm::DisplayHelp(argv[0]);
- return 1;
- }
- }
-
- if (config.mode == lm::MODE_UNSET) {
- lm::DisplayHelp(argv[0]);
- return 1;
- }
-
- if (config.phrase && config.mode != lm::MODE_UNION && config.mode != lm::MODE_MULTIPLE) {
- std::cerr << "Phrase constraint currently only works in multiple or union mode. If you really need it for single, put everything on one line and use union." << std::endl;
- return 1;
- }
-
- bool cmd_is_model = true;
- const char *cmd_input = argv[argc - 2];
- if (!strncmp(cmd_input, "vocab:", 6)) {
- cmd_is_model = false;
- cmd_input += 6;
- } else if (!strncmp(cmd_input, "model:", 6)) {
- cmd_input += 6;
- } else if (strchr(cmd_input, ':')) {
- std::cerr << "Specify vocab: or model: before the input file name, not " << cmd_input << std::endl;
- return 1;
- } else {
- std::cerr << "Assuming that " << cmd_input << " is a model file" << std::endl;
- }
- std::ifstream cmd_file;
- std::istream *vocab;
- if (cmd_is_model) {
- vocab = &std::cin;
- } else {
- cmd_file.open(cmd_input, std::ios::in);
- UTIL_THROW_IF(!cmd_file, util::ErrnoException, "Failed to open " << cmd_input);
- vocab = &cmd_file;
- }
-
- util::FilePiece model(cmd_is_model ? util::OpenReadOrThrow(cmd_input) : 0, cmd_is_model ? cmd_input : NULL, &std::cerr);
-
- if (config.format == lm::FORMAT_ARPA) {
- lm::DispatchFilterModes<lm::ARPAFormat>(config, *vocab, model, argv[argc - 1]);
- } else if (config.format == lm::FORMAT_COUNT) {
- lm::DispatchFilterModes<lm::CountFormat>(config, *vocab, model, argv[argc - 1]);
- }
- return 0;
- } catch (const std::exception &e) {
- std::cerr << e.what() << std::endl;
- return 1;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/filter/format.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/filter/format.hh b/ext/kenlm/lm/filter/format.hh
deleted file mode 100644
index d453f05..0000000
--- a/ext/kenlm/lm/filter/format.hh
+++ /dev/null
@@ -1,250 +0,0 @@
-#ifndef LM_FILTER_FORMAT_H
-#define LM_FILTER_FORMAT_H
-
-#include "lm/filter/arpa_io.hh"
-#include "lm/filter/count_io.hh"
-
-#include <boost/lexical_cast.hpp>
-#include <boost/ptr_container/ptr_vector.hpp>
-
-#include <iosfwd>
-
-namespace lm {
-
-template <class Single> class MultipleOutput {
- private:
- typedef boost::ptr_vector<Single> Singles;
- typedef typename Singles::iterator SinglesIterator;
-
- public:
- MultipleOutput(const char *prefix, size_t number) {
- files_.reserve(number);
- std::string tmp;
- for (unsigned int i = 0; i < number; ++i) {
- tmp = prefix;
- tmp += boost::lexical_cast<std::string>(i);
- files_.push_back(new Single(tmp.c_str()));
- }
- }
-
- void AddNGram(const StringPiece &line) {
- for (SinglesIterator i = files_.begin(); i != files_.end(); ++i)
- i->AddNGram(line);
- }
-
- template <class Iterator> void AddNGram(const Iterator &begin, const Iterator &end, const StringPiece &line) {
- for (SinglesIterator i = files_.begin(); i != files_.end(); ++i)
- i->AddNGram(begin, end, line);
- }
-
- void SingleAddNGram(size_t offset, const StringPiece &line) {
- files_[offset].AddNGram(line);
- }
-
- template <class Iterator> void SingleAddNGram(size_t offset, const Iterator &begin, const Iterator &end, const StringPiece &line) {
- files_[offset].AddNGram(begin, end, line);
- }
-
- protected:
- Singles files_;
-};
-
-class MultipleARPAOutput : public MultipleOutput<ARPAOutput> {
- public:
- MultipleARPAOutput(const char *prefix, size_t number) : MultipleOutput<ARPAOutput>(prefix, number) {}
-
- void ReserveForCounts(std::streampos reserve) {
- for (boost::ptr_vector<ARPAOutput>::iterator i = files_.begin(); i != files_.end(); ++i)
- i->ReserveForCounts(reserve);
- }
-
- void BeginLength(unsigned int length) {
- for (boost::ptr_vector<ARPAOutput>::iterator i = files_.begin(); i != files_.end(); ++i)
- i->BeginLength(length);
- }
-
- void EndLength(unsigned int length) {
- for (boost::ptr_vector<ARPAOutput>::iterator i = files_.begin(); i != files_.end(); ++i)
- i->EndLength(length);
- }
-
- void Finish() {
- for (boost::ptr_vector<ARPAOutput>::iterator i = files_.begin(); i != files_.end(); ++i)
- i->Finish();
- }
-};
-
-template <class Filter, class Output> class DispatchInput {
- public:
- DispatchInput(Filter &filter, Output &output) : filter_(filter), output_(output) {}
-
-/* template <class Iterator> void AddNGram(const Iterator &begin, const Iterator &end, const StringPiece &line) {
- filter_.AddNGram(begin, end, line, output_);
- }*/
-
- void AddNGram(const StringPiece &ngram, const StringPiece &line) {
- filter_.AddNGram(ngram, line, output_);
- }
-
- protected:
- Filter &filter_;
- Output &output_;
-};
-
-template <class Filter, class Output> class DispatchARPAInput : public DispatchInput<Filter, Output> {
- private:
- typedef DispatchInput<Filter, Output> B;
-
- public:
- DispatchARPAInput(Filter &filter, Output &output) : B(filter, output) {}
-
- void ReserveForCounts(std::streampos reserve) { B::output_.ReserveForCounts(reserve); }
- void BeginLength(unsigned int length) { B::output_.BeginLength(length); }
-
- void EndLength(unsigned int length) {
- B::filter_.Flush();
- B::output_.EndLength(length);
- }
- void Finish() { B::output_.Finish(); }
-};
-
-struct ARPAFormat {
- typedef ARPAOutput Output;
- typedef MultipleARPAOutput Multiple;
- static void Copy(util::FilePiece &in, Output &out) {
- ReadARPA(in, out);
- }
- template <class Filter, class Out> static void RunFilter(util::FilePiece &in, Filter &filter, Out &output) {
- DispatchARPAInput<Filter, Out> dispatcher(filter, output);
- ReadARPA(in, dispatcher);
- }
-};
-
-struct CountFormat {
- typedef CountOutput Output;
- typedef MultipleOutput<Output> Multiple;
- static void Copy(util::FilePiece &in, Output &out) {
- ReadCount(in, out);
- }
- template <class Filter, class Out> static void RunFilter(util::FilePiece &in, Filter &filter, Out &output) {
- DispatchInput<Filter, Out> dispatcher(filter, output);
- ReadCount(in, dispatcher);
- }
-};
-
-/* For multithreading, the buffer classes hold batches of filter inputs and
- * outputs in memory. The strings get reused a lot, so keep them around
- * instead of clearing each time.
- */
-class InputBuffer {
- public:
- InputBuffer() : actual_(0) {}
-
- void Reserve(size_t size) { lines_.reserve(size); }
-
- template <class Output> void AddNGram(const StringPiece &ngram, const StringPiece &line, Output &output) {
- if (lines_.size() == actual_) lines_.resize(lines_.size() + 1);
- // TODO avoid this copy.
- std::string &copied = lines_[actual_].line;
- copied.assign(line.data(), line.size());
- lines_[actual_].ngram.set(copied.data() + (ngram.data() - line.data()), ngram.size());
- ++actual_;
- }
-
- template <class Filter, class Output> void CallFilter(Filter &filter, Output &output) const {
- for (std::vector<Line>::const_iterator i = lines_.begin(); i != lines_.begin() + actual_; ++i) {
- filter.AddNGram(i->ngram, i->line, output);
- }
- }
-
- void Clear() { actual_ = 0; }
- bool Empty() { return actual_ == 0; }
- size_t Size() { return actual_; }
-
- private:
- struct Line {
- std::string line;
- StringPiece ngram;
- };
-
- size_t actual_;
-
- std::vector<Line> lines_;
-};
-
-class BinaryOutputBuffer {
- public:
- BinaryOutputBuffer() {}
-
- void Reserve(size_t size) {
- lines_.reserve(size);
- }
-
- void AddNGram(const StringPiece &line) {
- lines_.push_back(line);
- }
-
- template <class Output> void Flush(Output &output) {
- for (std::vector<StringPiece>::const_iterator i = lines_.begin(); i != lines_.end(); ++i) {
- output.AddNGram(*i);
- }
- lines_.clear();
- }
-
- private:
- std::vector<StringPiece> lines_;
-};
-
-class MultipleOutputBuffer {
- public:
- MultipleOutputBuffer() : last_(NULL) {}
-
- void Reserve(size_t size) {
- annotated_.reserve(size);
- }
-
- void AddNGram(const StringPiece &line) {
- annotated_.resize(annotated_.size() + 1);
- annotated_.back().line = line;
- }
-
- void SingleAddNGram(size_t offset, const StringPiece &line) {
- if ((line.data() == last_.data()) && (line.length() == last_.length())) {
- annotated_.back().systems.push_back(offset);
- } else {
- annotated_.resize(annotated_.size() + 1);
- annotated_.back().systems.push_back(offset);
- annotated_.back().line = line;
- last_ = line;
- }
- }
-
- template <class Output> void Flush(Output &output) {
- for (std::vector<Annotated>::const_iterator i = annotated_.begin(); i != annotated_.end(); ++i) {
- if (i->systems.empty()) {
- output.AddNGram(i->line);
- } else {
- for (std::vector<size_t>::const_iterator j = i->systems.begin(); j != i->systems.end(); ++j) {
- output.SingleAddNGram(*j, i->line);
- }
- }
- }
- annotated_.clear();
- }
-
- private:
- struct Annotated {
- // If this is empty, send to all systems.
- // A filter should never send to all systems and send to a single one.
- std::vector<size_t> systems;
- StringPiece line;
- };
-
- StringPiece last_;
-
- std::vector<Annotated> annotated_;
-};
-
-} // namespace lm
-
-#endif // LM_FILTER_FORMAT_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/filter/phrase.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/filter/phrase.cc b/ext/kenlm/lm/filter/phrase.cc
deleted file mode 100644
index d8260d5..0000000
--- a/ext/kenlm/lm/filter/phrase.cc
+++ /dev/null
@@ -1,292 +0,0 @@
-#include "lm/filter/phrase.hh"
-
-#include "lm/filter/format.hh"
-
-#include <algorithm>
-#include <functional>
-#include <iostream>
-#include <queue>
-#include <string>
-#include <vector>
-
-#include <cctype>
-
-namespace lm {
-namespace phrase {
-
-unsigned int ReadMultiple(std::istream &in, Substrings &out) {
- bool sentence_content = false;
- unsigned int sentence_id = 0;
- std::vector<Hash> phrase;
- std::string word;
- while (in) {
- char c;
- // Gather a word.
- while (!isspace(c = in.get()) && in) word += c;
- // Treat EOF like a newline.
- if (!in) c = '\n';
- // Add the word to the phrase.
- if (!word.empty()) {
- phrase.push_back(util::MurmurHashNative(word.data(), word.size()));
- word.clear();
- }
- if (c == ' ') continue;
- // It's more than just a space. Close out the phrase.
- if (!phrase.empty()) {
- sentence_content = true;
- out.AddPhrase(sentence_id, phrase.begin(), phrase.end());
- phrase.clear();
- }
- if (c == '\t' || c == '\v') continue;
- // It's more than a space or tab: a newline.
- if (sentence_content) {
- ++sentence_id;
- sentence_content = false;
- }
- }
- if (!in.eof()) in.exceptions(std::istream::failbit | std::istream::badbit);
- return sentence_id + sentence_content;
-}
-
-namespace {
-typedef unsigned int Sentence;
-typedef std::vector<Sentence> Sentences;
-} // namespace
-
-namespace detail {
-
-const StringPiece kEndSentence("</s>");
-
-class Arc {
- public:
- Arc() {}
-
- // For arcs from one vertex to another.
- void SetPhrase(detail::Vertex &from, detail::Vertex &to, const Sentences &intersect) {
- Set(to, intersect);
- from_ = &from;
- }
-
- /* For arcs from before the n-gram begins to somewhere in the n-gram (right
- * aligned). These have no from_ vertex; it implictly matches every
- * sentence. This also handles when the n-gram is a substring of a phrase.
- */
- void SetRight(detail::Vertex &to, const Sentences &complete) {
- Set(to, complete);
- from_ = NULL;
- }
-
- Sentence Current() const {
- return *current_;
- }
-
- bool Empty() const {
- return current_ == last_;
- }
-
- /* When this function returns:
- * If Empty() then there's nothing left from this intersection.
- *
- * If Current() == to then to is part of the intersection.
- *
- * Otherwise, Current() > to. In this case, to is not part of the
- * intersection and neither is anything < Current(). To determine if
- * any value >= Current() is in the intersection, call LowerBound again
- * with the value.
- */
- void LowerBound(const Sentence to);
-
- private:
- void Set(detail::Vertex &to, const Sentences &sentences);
-
- const Sentence *current_;
- const Sentence *last_;
- detail::Vertex *from_;
-};
-
-struct ArcGreater : public std::binary_function<const Arc *, const Arc *, bool> {
- bool operator()(const Arc *first, const Arc *second) const {
- return first->Current() > second->Current();
- }
-};
-
-class Vertex {
- public:
- Vertex() : current_(0) {}
-
- Sentence Current() const {
- return current_;
- }
-
- bool Empty() const {
- return incoming_.empty();
- }
-
- void LowerBound(const Sentence to);
-
- private:
- friend class Arc;
-
- void AddIncoming(Arc *arc) {
- if (!arc->Empty()) incoming_.push(arc);
- }
-
- unsigned int current_;
- std::priority_queue<Arc*, std::vector<Arc*>, ArcGreater> incoming_;
-};
-
-void Arc::LowerBound(const Sentence to) {
- current_ = std::lower_bound(current_, last_, to);
- // If *current_ > to, don't advance from_. The intervening values of
- // from_ may be useful for another one of its outgoing arcs.
- if (!from_ || Empty() || (Current() > to)) return;
- assert(Current() == to);
- from_->LowerBound(to);
- if (from_->Empty()) {
- current_ = last_;
- return;
- }
- assert(from_->Current() >= to);
- if (from_->Current() > to) {
- current_ = std::lower_bound(current_ + 1, last_, from_->Current());
- }
-}
-
-void Arc::Set(Vertex &to, const Sentences &sentences) {
- current_ = &*sentences.begin();
- last_ = &*sentences.end();
- to.AddIncoming(this);
-}
-
-void Vertex::LowerBound(const Sentence to) {
- if (Empty()) return;
- // Union lower bound.
- while (true) {
- Arc *top = incoming_.top();
- if (top->Current() > to) {
- current_ = top->Current();
- return;
- }
- // If top->Current() == to, we still need to verify that's an actual
- // element and not just a bound.
- incoming_.pop();
- top->LowerBound(to);
- if (!top->Empty()) {
- incoming_.push(top);
- if (top->Current() == to) {
- current_ = to;
- return;
- }
- } else if (Empty()) {
- return;
- }
- }
-}
-
-} // namespace detail
-
-namespace {
-
-void BuildGraph(const Substrings &phrase, const std::vector<Hash> &hashes, detail::Vertex *const vertices, detail::Arc *free_arc) {
- using detail::Vertex;
- using detail::Arc;
- assert(!hashes.empty());
-
- const Hash *const first_word = &*hashes.begin();
- const Hash *const last_word = &*hashes.end() - 1;
-
- Hash hash = 0;
- const Sentences *found;
- // Phrases starting at or before the first word in the n-gram.
- {
- Vertex *vertex = vertices;
- for (const Hash *word = first_word; ; ++word, ++vertex) {
- hash = util::MurmurHashNative(&hash, sizeof(uint64_t), *word);
- // Now hash is [hashes.begin(), word].
- if (word == last_word) {
- if (phrase.FindSubstring(hash, found))
- (free_arc++)->SetRight(*vertex, *found);
- break;
- }
- if (!phrase.FindRight(hash, found)) break;
- (free_arc++)->SetRight(*vertex, *found);
- }
- }
-
- // Phrases starting at the second or later word in the n-gram.
- Vertex *vertex_from = vertices;
- for (const Hash *word_from = first_word + 1; word_from != &*hashes.end(); ++word_from, ++vertex_from) {
- hash = 0;
- Vertex *vertex_to = vertex_from + 1;
- for (const Hash *word_to = word_from; ; ++word_to, ++vertex_to) {
- // Notice that word_to and vertex_to have the same index.
- hash = util::MurmurHashNative(&hash, sizeof(uint64_t), *word_to);
- // Now hash covers [word_from, word_to].
- if (word_to == last_word) {
- if (phrase.FindLeft(hash, found))
- (free_arc++)->SetPhrase(*vertex_from, *vertex_to, *found);
- break;
- }
- if (!phrase.FindPhrase(hash, found)) break;
- (free_arc++)->SetPhrase(*vertex_from, *vertex_to, *found);
- }
- }
-}
-
-} // namespace
-
-namespace detail {
-
-// Here instead of header due to forward declaration.
-ConditionCommon::ConditionCommon(const Substrings &substrings) : substrings_(substrings) {}
-
-// Rest of the variables are temporaries anyway
-ConditionCommon::ConditionCommon(const ConditionCommon &from) : substrings_(from.substrings_) {}
-
-ConditionCommon::~ConditionCommon() {}
-
-detail::Vertex &ConditionCommon::MakeGraph() {
- assert(!hashes_.empty());
- vertices_.clear();
- vertices_.resize(hashes_.size());
- arcs_.clear();
- // One for every substring.
- arcs_.resize(((hashes_.size() + 1) * hashes_.size()) / 2);
- BuildGraph(substrings_, hashes_, &*vertices_.begin(), &*arcs_.begin());
- return vertices_[hashes_.size() - 1];
-}
-
-} // namespace detail
-
-bool Union::Evaluate() {
- detail::Vertex &last_vertex = MakeGraph();
- unsigned int lower = 0;
- while (true) {
- last_vertex.LowerBound(lower);
- if (last_vertex.Empty()) return false;
- if (last_vertex.Current() == lower) return true;
- lower = last_vertex.Current();
- }
-}
-
-template <class Output> void Multiple::Evaluate(const StringPiece &line, Output &output) {
- detail::Vertex &last_vertex = MakeGraph();
- unsigned int lower = 0;
- while (true) {
- last_vertex.LowerBound(lower);
- if (last_vertex.Empty()) return;
- if (last_vertex.Current() == lower) {
- output.SingleAddNGram(lower, line);
- ++lower;
- } else {
- lower = last_vertex.Current();
- }
- }
-}
-
-template void Multiple::Evaluate<CountFormat::Multiple>(const StringPiece &line, CountFormat::Multiple &output);
-template void Multiple::Evaluate<ARPAFormat::Multiple>(const StringPiece &line, ARPAFormat::Multiple &output);
-template void Multiple::Evaluate<MultipleOutputBuffer>(const StringPiece &line, MultipleOutputBuffer &output);
-
-} // namespace phrase
-} // namespace lm
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/filter/phrase.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/filter/phrase.hh b/ext/kenlm/lm/filter/phrase.hh
deleted file mode 100644
index 5227ab2..0000000
--- a/ext/kenlm/lm/filter/phrase.hh
+++ /dev/null
@@ -1,168 +0,0 @@
-#ifndef LM_FILTER_PHRASE_H
-#define LM_FILTER_PHRASE_H
-
-#include "util/murmur_hash.hh"
-#include "util/string_piece.hh"
-#include "util/tokenize_piece.hh"
-
-#include <boost/unordered_map.hpp>
-
-#include <iosfwd>
-#include <vector>
-
-#define LM_FILTER_PHRASE_METHOD(caps, lower) \
-bool Find##caps(Hash key, const std::vector<unsigned int> *&out) const {\
- Table::const_iterator i(table_.find(key));\
- if (i==table_.end()) return false; \
- out = &i->second.lower; \
- return true; \
-}
-
-namespace lm {
-namespace phrase {
-
-typedef uint64_t Hash;
-
-class Substrings {
- private:
- /* This is the value in a hash table where the key is a string. It indicates
- * four sets of sentences:
- * substring is sentences with a phrase containing the key as a substring.
- * left is sentencess with a phrase that begins with the key (left aligned).
- * right is sentences with a phrase that ends with the key (right aligned).
- * phrase is sentences where the key is a phrase.
- * Each set is encoded as a vector of sentence ids in increasing order.
- */
- struct SentenceRelation {
- std::vector<unsigned int> substring, left, right, phrase;
- };
- /* Most of the CPU is hash table lookups, so let's not complicate it with
- * vector equality comparisons. If a collision happens, the SentenceRelation
- * structure will contain the union of sentence ids over the colliding strings.
- * In that case, the filter will be slightly more permissive.
- * The key here is the same as boost's hash of std::vector<std::string>.
- */
- typedef boost::unordered_map<Hash, SentenceRelation> Table;
-
- public:
- Substrings() {}
-
- /* If the string isn't a substring of any phrase, return NULL. Otherwise,
- * return a pointer to std::vector<unsigned int> listing sentences with
- * matching phrases. This set may be empty for Left, Right, or Phrase.
- * Example: const std::vector<unsigned int> *FindSubstring(Hash key)
- */
- LM_FILTER_PHRASE_METHOD(Substring, substring)
- LM_FILTER_PHRASE_METHOD(Left, left)
- LM_FILTER_PHRASE_METHOD(Right, right)
- LM_FILTER_PHRASE_METHOD(Phrase, phrase)
-
-#pragma GCC diagnostic ignored "-Wuninitialized" // end != finish so there's always an initialization
- // sentence_id must be non-decreasing. Iterators are over words in the phrase.
- template <class Iterator> void AddPhrase(unsigned int sentence_id, const Iterator &begin, const Iterator &end) {
- // Iterate over all substrings.
- for (Iterator start = begin; start != end; ++start) {
- Hash hash = 0;
- SentenceRelation *relation;
- for (Iterator finish = start; finish != end; ++finish) {
- hash = util::MurmurHashNative(&hash, sizeof(uint64_t), *finish);
- // Now hash is of [start, finish].
- relation = &table_[hash];
- AppendSentence(relation->substring, sentence_id);
- if (start == begin) AppendSentence(relation->left, sentence_id);
- }
- AppendSentence(relation->right, sentence_id);
- if (start == begin) AppendSentence(relation->phrase, sentence_id);
- }
- }
-
- private:
- void AppendSentence(std::vector<unsigned int> &vec, unsigned int sentence_id) {
- if (vec.empty() || vec.back() != sentence_id) vec.push_back(sentence_id);
- }
-
- Table table_;
-};
-
-// Read a file with one sentence per line containing tab-delimited phrases of
-// space-separated words.
-unsigned int ReadMultiple(std::istream &in, Substrings &out);
-
-namespace detail {
-extern const StringPiece kEndSentence;
-
-template <class Iterator> void MakeHashes(Iterator i, const Iterator &end, std::vector<Hash> &hashes) {
- hashes.clear();
- if (i == end) return;
- // TODO: check strict phrase boundaries after <s> and before </s>. For now, just skip tags.
- if ((i->data()[0] == '<') && (i->data()[i->size() - 1] == '>')) {
- ++i;
- }
- for (; i != end && (*i != kEndSentence); ++i) {
- hashes.push_back(util::MurmurHashNative(i->data(), i->size()));
- }
-}
-
-class Vertex;
-class Arc;
-
-class ConditionCommon {
- protected:
- ConditionCommon(const Substrings &substrings);
- ConditionCommon(const ConditionCommon &from);
-
- ~ConditionCommon();
-
- detail::Vertex &MakeGraph();
-
- // Temporaries in PassNGram and Evaluate to avoid reallocation.
- std::vector<Hash> hashes_;
-
- private:
- std::vector<detail::Vertex> vertices_;
- std::vector<detail::Arc> arcs_;
-
- const Substrings &substrings_;
-};
-
-} // namespace detail
-
-class Union : public detail::ConditionCommon {
- public:
- explicit Union(const Substrings &substrings) : detail::ConditionCommon(substrings) {}
-
- template <class Iterator> bool PassNGram(const Iterator &begin, const Iterator &end) {
- detail::MakeHashes(begin, end, hashes_);
- return hashes_.empty() || Evaluate();
- }
-
- private:
- bool Evaluate();
-};
-
-class Multiple : public detail::ConditionCommon {
- public:
- explicit Multiple(const Substrings &substrings) : detail::ConditionCommon(substrings) {}
-
- template <class Iterator, class Output> void AddNGram(const Iterator &begin, const Iterator &end, const StringPiece &line, Output &output) {
- detail::MakeHashes(begin, end, hashes_);
- if (hashes_.empty()) {
- output.AddNGram(line);
- } else {
- Evaluate(line, output);
- }
- }
-
- template <class Output> void AddNGram(const StringPiece &ngram, const StringPiece &line, Output &output) {
- AddNGram(util::TokenIter<util::SingleCharacter, true>(ngram, ' '), util::TokenIter<util::SingleCharacter, true>::end(), line, output);
- }
-
- void Flush() const {}
-
- private:
- template <class Output> void Evaluate(const StringPiece &line, Output &output);
-};
-
-} // namespace phrase
-} // namespace lm
-#endif // LM_FILTER_PHRASE_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/filter/phrase_table_vocab_main.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/filter/phrase_table_vocab_main.cc b/ext/kenlm/lm/filter/phrase_table_vocab_main.cc
deleted file mode 100644
index 9ffa35f..0000000
--- a/ext/kenlm/lm/filter/phrase_table_vocab_main.cc
+++ /dev/null
@@ -1,165 +0,0 @@
-#include "util/file_stream.hh"
-#include "util/file_piece.hh"
-#include "util/murmur_hash.hh"
-#include "util/pool.hh"
-#include "util/string_piece.hh"
-#include "util/string_piece_hash.hh"
-#include "util/tokenize_piece.hh"
-
-#include <boost/unordered_map.hpp>
-#include <boost/unordered_set.hpp>
-
-#include <cstddef>
-#include <vector>
-
-namespace {
-
-struct MutablePiece {
- mutable StringPiece behind;
- bool operator==(const MutablePiece &other) const {
- return behind == other.behind;
- }
-};
-
-std::size_t hash_value(const MutablePiece &m) {
- return hash_value(m.behind);
-}
-
-class InternString {
- public:
- const char *Add(StringPiece str) {
- MutablePiece mut;
- mut.behind = str;
- std::pair<boost::unordered_set<MutablePiece>::iterator, bool> res(strs_.insert(mut));
- if (res.second) {
- void *mem = backing_.Allocate(str.size() + 1);
- memcpy(mem, str.data(), str.size());
- static_cast<char*>(mem)[str.size()] = 0;
- res.first->behind = StringPiece(static_cast<char*>(mem), str.size());
- }
- return res.first->behind.data();
- }
-
- private:
- util::Pool backing_;
- boost::unordered_set<MutablePiece> strs_;
-};
-
-class TargetWords {
- public:
- void Introduce(StringPiece source) {
- vocab_.resize(vocab_.size() + 1);
- std::vector<unsigned int> temp(1, vocab_.size() - 1);
- Add(temp, source);
- }
-
- void Add(const std::vector<unsigned int> &sentences, StringPiece target) {
- if (sentences.empty()) return;
- interns_.clear();
- for (util::TokenIter<util::SingleCharacter, true> i(target, ' '); i; ++i) {
- interns_.push_back(intern_.Add(*i));
- }
- for (std::vector<unsigned int>::const_iterator i(sentences.begin()); i != sentences.end(); ++i) {
- boost::unordered_set<const char *> &vocab = vocab_[*i];
- for (std::vector<const char *>::const_iterator j = interns_.begin(); j != interns_.end(); ++j) {
- vocab.insert(*j);
- }
- }
- }
-
- void Print() const {
- util::FileStream out(1);
- for (std::vector<boost::unordered_set<const char *> >::const_iterator i = vocab_.begin(); i != vocab_.end(); ++i) {
- for (boost::unordered_set<const char *>::const_iterator j = i->begin(); j != i->end(); ++j) {
- out << *j << ' ';
- }
- out << '\n';
- }
- }
-
- private:
- InternString intern_;
-
- std::vector<boost::unordered_set<const char *> > vocab_;
-
- // Temporary in Add.
- std::vector<const char *> interns_;
-};
-
-class Input {
- public:
- explicit Input(std::size_t max_length)
- : max_length_(max_length), sentence_id_(0), empty_() {}
-
- void AddSentence(StringPiece sentence, TargetWords &targets) {
- canonical_.clear();
- starts_.clear();
- starts_.push_back(0);
- for (util::TokenIter<util::AnyCharacter, true> i(sentence, StringPiece("\0 \t", 3)); i; ++i) {
- canonical_.append(i->data(), i->size());
- canonical_ += ' ';
- starts_.push_back(canonical_.size());
- }
- targets.Introduce(canonical_);
- for (std::size_t i = 0; i < starts_.size() - 1; ++i) {
- std::size_t subtract = starts_[i];
- const char *start = &canonical_[subtract];
- for (std::size_t j = i + 1; j < std::min(starts_.size(), i + max_length_ + 1); ++j) {
- map_[util::MurmurHash64A(start, &canonical_[starts_[j]] - start - 1)].push_back(sentence_id_);
- }
- }
- ++sentence_id_;
- }
-
- // Assumes single space-delimited phrase with no space at the beginning or end.
- const std::vector<unsigned int> &Matches(StringPiece phrase) const {
- Map::const_iterator i = map_.find(util::MurmurHash64A(phrase.data(), phrase.size()));
- return i == map_.end() ? empty_ : i->second;
- }
-
- private:
- const std::size_t max_length_;
-
- // hash of phrase is the key, array of sentences is the value.
- typedef boost::unordered_map<uint64_t, std::vector<unsigned int> > Map;
- Map map_;
-
- std::size_t sentence_id_;
-
- // Temporaries in AddSentence.
- std::string canonical_;
- std::vector<std::size_t> starts_;
-
- const std::vector<unsigned int> empty_;
-};
-
-} // namespace
-
-int main(int argc, char *argv[]) {
- if (argc != 2) {
- std::cerr << "Expected source text on the command line" << std::endl;
- return 1;
- }
- Input input(7);
- TargetWords targets;
- try {
- util::FilePiece inputs(argv[1], &std::cerr);
- while (true)
- input.AddSentence(inputs.ReadLine(), targets);
- } catch (const util::EndOfFileException &e) {}
-
- util::FilePiece table(0, NULL, &std::cerr);
- StringPiece line;
- const StringPiece pipes("|||");
- while (true) {
- try {
- line = table.ReadLine();
- } catch (const util::EndOfFileException &e) { break; }
- util::TokenIter<util::MultiCharacter> it(line, pipes);
- StringPiece source(*it);
- if (!source.empty() && source[source.size() - 1] == ' ')
- source.remove_suffix(1);
- targets.Add(input.Matches(source), *++it);
- }
- targets.Print();
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/filter/thread.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/filter/thread.hh b/ext/kenlm/lm/filter/thread.hh
deleted file mode 100644
index 88e069c..0000000
--- a/ext/kenlm/lm/filter/thread.hh
+++ /dev/null
@@ -1,167 +0,0 @@
-#ifndef LM_FILTER_THREAD_H
-#define LM_FILTER_THREAD_H
-
-#include "util/thread_pool.hh"
-
-#include <boost/utility/in_place_factory.hpp>
-
-#include <deque>
-#include <stack>
-
-namespace lm {
-
-template <class OutputBuffer> class ThreadBatch {
- public:
- ThreadBatch() {}
-
- void Reserve(size_t size) {
- input_.Reserve(size);
- output_.Reserve(size);
- }
-
- // File reading thread.
- InputBuffer &Fill(uint64_t sequence) {
- sequence_ = sequence;
- // Why wait until now to clear instead of after output? free in the same
- // thread as allocated.
- input_.Clear();
- return input_;
- }
-
- // Filter worker thread.
- template <class Filter> void CallFilter(Filter &filter) {
- input_.CallFilter(filter, output_);
- }
-
- uint64_t Sequence() const { return sequence_; }
-
- // File writing thread.
- template <class RealOutput> void Flush(RealOutput &output) {
- output_.Flush(output);
- }
-
- private:
- InputBuffer input_;
- OutputBuffer output_;
-
- uint64_t sequence_;
-};
-
-template <class Batch, class Filter> class FilterWorker {
- public:
- typedef Batch *Request;
-
- FilterWorker(const Filter &filter, util::PCQueue<Request> &done) : filter_(filter), done_(done) {}
-
- void operator()(Request request) {
- request->CallFilter(filter_);
- done_.Produce(request);
- }
-
- private:
- Filter filter_;
-
- util::PCQueue<Request> &done_;
-};
-
-// There should only be one OutputWorker.
-template <class Batch, class Output> class OutputWorker {
- public:
- typedef Batch *Request;
-
- OutputWorker(Output &output, util::PCQueue<Request> &done) : output_(output), done_(done), base_sequence_(0) {}
-
- void operator()(Request request) {
- assert(request->Sequence() >= base_sequence_);
- // Assemble the output in order.
- uint64_t pos = request->Sequence() - base_sequence_;
- if (pos >= ordering_.size()) {
- ordering_.resize(pos + 1, NULL);
- }
- ordering_[pos] = request;
- while (!ordering_.empty() && ordering_.front()) {
- ordering_.front()->Flush(output_);
- done_.Produce(ordering_.front());
- ordering_.pop_front();
- ++base_sequence_;
- }
- }
-
- private:
- Output &output_;
-
- util::PCQueue<Request> &done_;
-
- std::deque<Request> ordering_;
-
- uint64_t base_sequence_;
-};
-
-template <class Filter, class OutputBuffer, class RealOutput> class Controller : boost::noncopyable {
- private:
- typedef ThreadBatch<OutputBuffer> Batch;
-
- public:
- Controller(size_t batch_size, size_t queue, size_t workers, const Filter &filter, RealOutput &output)
- : batch_size_(batch_size), queue_size_(queue),
- batches_(queue),
- to_read_(queue),
- output_(queue, 1, boost::in_place(boost::ref(output), boost::ref(to_read_)), NULL),
- filter_(queue, workers, boost::in_place(boost::ref(filter), boost::ref(output_.In())), NULL),
- sequence_(0) {
- for (size_t i = 0; i < queue; ++i) {
- batches_[i].Reserve(batch_size);
- local_read_.push(&batches_[i]);
- }
- NewInput();
- }
-
- void AddNGram(const StringPiece &ngram, const StringPiece &line, RealOutput &output) {
- input_->AddNGram(ngram, line, output);
- if (input_->Size() == batch_size_) {
- FlushInput();
- NewInput();
- }
- }
-
- void Flush() {
- FlushInput();
- while (local_read_.size() < queue_size_) {
- MoveRead();
- }
- NewInput();
- }
-
- private:
- void FlushInput() {
- if (input_->Empty()) return;
- filter_.Produce(local_read_.top());
- local_read_.pop();
- if (local_read_.empty()) MoveRead();
- }
-
- void NewInput() {
- input_ = &local_read_.top()->Fill(sequence_++);
- }
-
- void MoveRead() {
- local_read_.push(to_read_.Consume());
- }
-
- const size_t batch_size_;
- const size_t queue_size_;
-
- std::vector<Batch> batches_;
-
- util::PCQueue<Batch*> to_read_;
- std::stack<Batch*> local_read_;
- util::ThreadPool<OutputWorker<Batch, RealOutput> > output_;
- util::ThreadPool<FilterWorker<Batch, Filter> > filter_;
-
- uint64_t sequence_;
- InputBuffer *input_;
-};
-
-} // namespace lm
-
-#endif // LM_FILTER_THREAD_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/filter/vocab.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/filter/vocab.cc b/ext/kenlm/lm/filter/vocab.cc
deleted file mode 100644
index 2aca4fc..0000000
--- a/ext/kenlm/lm/filter/vocab.cc
+++ /dev/null
@@ -1,53 +0,0 @@
-#include "lm/filter/vocab.hh"
-
-#include <istream>
-#include <iostream>
-
-#include <cctype>
-
-namespace lm {
-namespace vocab {
-
-void ReadSingle(std::istream &in, boost::unordered_set<std::string> &out) {
- in.exceptions(std::istream::badbit);
- std::string word;
- while (in >> word) {
- out.insert(word);
- }
-}
-
-namespace {
-bool IsLineEnd(std::istream &in) {
- int got;
- do {
- got = in.get();
- if (!in) return true;
- if (got == '\n') return true;
- } while (isspace(got));
- in.unget();
- return false;
-}
-}// namespace
-
-// Read space separated words in enter separated lines. These lines can be
-// very long, so don't read an entire line at a time.
-unsigned int ReadMultiple(std::istream &in, boost::unordered_map<std::string, std::vector<unsigned int> > &out) {
- in.exceptions(std::istream::badbit);
- unsigned int sentence = 0;
- bool used_id = false;
- std::string word;
- while (in >> word) {
- used_id = true;
- std::vector<unsigned int> &posting = out[word];
- if (posting.empty() || (posting.back() != sentence))
- posting.push_back(sentence);
- if (IsLineEnd(in)) {
- ++sentence;
- used_id = false;
- }
- }
- return sentence + used_id;
-}
-
-} // namespace vocab
-} // namespace lm
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/filter/vocab.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/filter/vocab.hh b/ext/kenlm/lm/filter/vocab.hh
deleted file mode 100644
index 397a932..0000000
--- a/ext/kenlm/lm/filter/vocab.hh
+++ /dev/null
@@ -1,133 +0,0 @@
-#ifndef LM_FILTER_VOCAB_H
-#define LM_FILTER_VOCAB_H
-
-// Vocabulary-based filters for language models.
-
-#include "util/multi_intersection.hh"
-#include "util/string_piece.hh"
-#include "util/string_piece_hash.hh"
-#include "util/tokenize_piece.hh"
-
-#include <boost/noncopyable.hpp>
-#include <boost/range/iterator_range.hpp>
-#include <boost/unordered/unordered_map.hpp>
-#include <boost/unordered/unordered_set.hpp>
-
-#include <string>
-#include <vector>
-
-namespace lm {
-namespace vocab {
-
-void ReadSingle(std::istream &in, boost::unordered_set<std::string> &out);
-
-// Read one sentence vocabulary per line. Return the number of sentences.
-unsigned int ReadMultiple(std::istream &in, boost::unordered_map<std::string, std::vector<unsigned int> > &out);
-
-/* Is this a special tag like <s> or <UNK>? This actually includes anything
- * surrounded with < and >, which most tokenizers separate for real words, so
- * this should not catch real words as it looks at a single token.
- */
-inline bool IsTag(const StringPiece &value) {
- // The parser should never give an empty string.
- assert(!value.empty());
- return (value.data()[0] == '<' && value.data()[value.size() - 1] == '>');
-}
-
-class Single {
- public:
- typedef boost::unordered_set<std::string> Words;
-
- explicit Single(const Words &vocab) : vocab_(vocab) {}
-
- template <class Iterator> bool PassNGram(const Iterator &begin, const Iterator &end) {
- for (Iterator i = begin; i != end; ++i) {
- if (IsTag(*i)) continue;
- if (FindStringPiece(vocab_, *i) == vocab_.end()) return false;
- }
- return true;
- }
-
- private:
- const Words &vocab_;
-};
-
-class Union {
- public:
- typedef boost::unordered_map<std::string, std::vector<unsigned int> > Words;
-
- explicit Union(const Words &vocabs) : vocabs_(vocabs) {}
-
- template <class Iterator> bool PassNGram(const Iterator &begin, const Iterator &end) {
- sets_.clear();
-
- for (Iterator i(begin); i != end; ++i) {
- if (IsTag(*i)) continue;
- Words::const_iterator found(FindStringPiece(vocabs_, *i));
- if (vocabs_.end() == found) return false;
- sets_.push_back(boost::iterator_range<const unsigned int*>(&*found->second.begin(), &*found->second.end()));
- }
- return (sets_.empty() || util::FirstIntersection(sets_));
- }
-
- private:
- const Words &vocabs_;
-
- std::vector<boost::iterator_range<const unsigned int*> > sets_;
-};
-
-class Multiple {
- public:
- typedef boost::unordered_map<std::string, std::vector<unsigned int> > Words;
-
- Multiple(const Words &vocabs) : vocabs_(vocabs) {}
-
- private:
- // Callback from AllIntersection that does AddNGram.
- template <class Output> class Callback {
- public:
- Callback(Output &out, const StringPiece &line) : out_(out), line_(line) {}
-
- void operator()(unsigned int index) {
- out_.SingleAddNGram(index, line_);
- }
-
- private:
- Output &out_;
- const StringPiece &line_;
- };
-
- public:
- template <class Iterator, class Output> void AddNGram(const Iterator &begin, const Iterator &end, const StringPiece &line, Output &output) {
- sets_.clear();
- for (Iterator i(begin); i != end; ++i) {
- if (IsTag(*i)) continue;
- Words::const_iterator found(FindStringPiece(vocabs_, *i));
- if (vocabs_.end() == found) return;
- sets_.push_back(boost::iterator_range<const unsigned int*>(&*found->second.begin(), &*found->second.end()));
- }
- if (sets_.empty()) {
- output.AddNGram(line);
- return;
- }
-
- Callback<Output> cb(output, line);
- util::AllIntersection(sets_, cb);
- }
-
- template <class Output> void AddNGram(const StringPiece &ngram, const StringPiece &line, Output &output) {
- AddNGram(util::TokenIter<util::SingleCharacter, true>(ngram, ' '), util::TokenIter<util::SingleCharacter, true>::end(), line, output);
- }
-
- void Flush() const {}
-
- private:
- const Words &vocabs_;
-
- std::vector<boost::iterator_range<const unsigned int*> > sets_;
-};
-
-} // namespace vocab
-} // namespace lm
-
-#endif // LM_FILTER_VOCAB_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/filter/wrapper.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/filter/wrapper.hh b/ext/kenlm/lm/filter/wrapper.hh
deleted file mode 100644
index 227ec8e..0000000
--- a/ext/kenlm/lm/filter/wrapper.hh
+++ /dev/null
@@ -1,56 +0,0 @@
-#ifndef LM_FILTER_WRAPPER_H
-#define LM_FILTER_WRAPPER_H
-
-#include "util/string_piece.hh"
-
-#include <algorithm>
-#include <string>
-#include <vector>
-
-namespace lm {
-
-// Provide a single-output filter with the same interface as a
-// multiple-output filter so clients code against one interface.
-template <class Binary> class BinaryFilter {
- public:
- // Binary modes are just references (and a set) and it makes the API cleaner to copy them.
- explicit BinaryFilter(Binary binary) : binary_(binary) {}
-
- template <class Iterator, class Output> void AddNGram(const Iterator &begin, const Iterator &end, const StringPiece &line, Output &output) {
- if (binary_.PassNGram(begin, end))
- output.AddNGram(line);
- }
-
- template <class Output> void AddNGram(const StringPiece &ngram, const StringPiece &line, Output &output) {
- AddNGram(util::TokenIter<util::SingleCharacter, true>(ngram, ' '), util::TokenIter<util::SingleCharacter, true>::end(), line, output);
- }
-
- void Flush() const {}
-
- private:
- Binary binary_;
-};
-
-// Wrap another filter to pay attention only to context words
-template <class FilterT> class ContextFilter {
- public:
- typedef FilterT Filter;
-
- explicit ContextFilter(Filter &backend) : backend_(backend) {}
-
- template <class Output> void AddNGram(const StringPiece &ngram, const StringPiece &line, Output &output) {
- // Find beginning of string or last space.
- const char *last_space;
- for (last_space = ngram.data() + ngram.size() - 1; last_space > ngram.data() && *last_space != ' '; --last_space) {}
- backend_.AddNGram(StringPiece(ngram.data(), last_space - ngram.data()), line, output);
- }
-
- void Flush() const {}
-
- private:
- Filter backend_;
-};
-
-} // namespace lm
-
-#endif // LM_FILTER_WRAPPER_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/fragment_main.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/fragment_main.cc b/ext/kenlm/lm/fragment_main.cc
deleted file mode 100644
index 0267cd4..0000000
--- a/ext/kenlm/lm/fragment_main.cc
+++ /dev/null
@@ -1,37 +0,0 @@
-#include "lm/binary_format.hh"
-#include "lm/model.hh"
-#include "lm/left.hh"
-#include "util/tokenize_piece.hh"
-
-template <class Model> void Query(const char *name) {
- Model model(name);
- std::string line;
- lm::ngram::ChartState ignored;
- while (getline(std::cin, line)) {
- lm::ngram::RuleScore<Model> scorer(model, ignored);
- for (util::TokenIter<util::SingleCharacter, true> i(line, ' '); i; ++i) {
- scorer.Terminal(model.GetVocabulary().Index(*i));
- }
- std::cout << scorer.Finish() << '\n';
- }
-}
-
-int main(int argc, char *argv[]) {
- if (argc != 2) {
- std::cerr << "Expected model file name." << std::endl;
- return 1;
- }
- const char *name = argv[1];
- lm::ngram::ModelType model_type = lm::ngram::PROBING;
- lm::ngram::RecognizeBinary(name, model_type);
- switch (model_type) {
- case lm::ngram::PROBING:
- Query<lm::ngram::ProbingModel>(name);
- break;
- case lm::ngram::REST_PROBING:
- Query<lm::ngram::RestProbingModel>(name);
- break;
- default:
- std::cerr << "Model type not supported yet." << std::endl;
- }
-}
[37/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/msvc.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/msvc.jam b/ext/kenlm/jam-files/boost-build/tools/msvc.jam
deleted file mode 100644
index 6ecd033..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/msvc.jam
+++ /dev/null
@@ -1,1435 +0,0 @@
-# Copyright (c) 2003 David Abrahams
-# Copyright (c) 2005 Vladimir Prus
-# Copyright (c) 2005 Alexey Pakhunov
-# Copyright (c) 2006 Bojan Resnik
-# Copyright (c) 2006 Ilya Sokolov
-# Copyright (c) 2007 Rene Rivera
-# Copyright (c) 2008 Jurko Gospodnetic
-#
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-################################################################################
-#
-# MSVC Boost Build toolset module.
-# --------------------------------
-#
-# All toolset versions need to have their location either auto-detected or
-# explicitly specified except for the special 'default' version that expects the
-# environment to find the needed tools or report an error.
-#
-################################################################################
-
-import "class" : new ;
-import common ;
-import feature ;
-import generators ;
-import mc ;
-import midl ;
-import os ;
-import path ;
-import pch ;
-import property ;
-import rc ;
-import toolset ;
-import type ;
-
-
-type.register MANIFEST : manifest ;
-feature.feature embed-manifest : on off : incidental propagated ;
-
-type.register PDB : pdb ;
-
-
-################################################################################
-#
-# Public rules.
-#
-################################################################################
-
-# Initialize a specific toolset version configuration. As the result, path to
-# compiler and, possible, program names are set up, and will be used when that
-# version of compiler is requested. For example, you might have:
-#
-# using msvc : 6.5 : cl.exe ;
-# using msvc : 7.0 : Y:/foo/bar/cl.exe ;
-#
-# The version parameter may be ommited:
-#
-# using msvc : : Z:/foo/bar/cl.exe ;
-#
-# The following keywords have special meanings when specified as versions:
-# - all - all detected but not yet used versions will be marked as used
-# with their default options.
-# - default - this is an equivalent to an empty version.
-#
-# Depending on a supplied version, detected configurations and presence 'cl.exe'
-# in the path different results may be achieved. The following table describes
-# the possible scenarios:
-#
-# Nothing "x.y"
-# Passed Nothing "x.y" detected, detected,
-# version detected detected cl.exe in path cl.exe in path
-#
-# default Error Use "x.y" Create "default" Use "x.y"
-# all None Use all None Use all
-# x.y - Use "x.y" - Use "x.y"
-# a.b Error Error Create "a.b" Create "a.b"
-#
-# "x.y" - refers to a detected version;
-# "a.b" - refers to an undetected version.
-#
-# FIXME: Currently the command parameter and the <compiler> property parameter
-# seem to overlap in duties. Remove this duplication. This seems to be related
-# to why someone started preparing to replace init with configure rules.
-#
-rule init (
- # The msvc version being configured. When omitted the tools invoked when no
- # explicit version is given will be configured.
- version ?
-
- # The command used to invoke the compiler. If not specified:
- # - if version is given, default location for that version will be
- # searched
- #
- # - if version is not given, default locations for MSVC 9.0, 8.0, 7.1, 7.0
- # and 6.* will be searched
- #
- # - if compiler is not found in the default locations, PATH will be
- # searched.
- : command *
-
- # Options may include:
- #
- # All options shared by multiple toolset types as handled by the
- # common.handle-options() rule, e.g. <cflags>, <compileflags>, <cxxflags>,
- # <fflags> & <linkflags>.
- #
- # <assembler>
- # <compiler>
- # <idl-compiler>
- # <linker>
- # <mc-compiler>
- # <resource-compiler>
- # Exact tool names to be used by this msvc toolset configuration.
- #
- # <compiler-filter>
- # Command through which to pipe the output of running the compiler.
- # For example to pass the output to STLfilt.
- #
- # <setup>
- # Global setup command to invoke before running any of the msvc tools.
- # It will be passed additional option parameters depending on the actual
- # target platform.
- #
- # <setup-amd64>
- # <setup-i386>
- # <setup-ia64>
- # <setup-arm>
- # Platform specific setup command to invoke before running any of the
- # msvc tools used when builing a target for a specific platform, e.g.
- # when building a 32 or 64 bit executable.
- : options *
-)
-{
- if $(command)
- {
- options += <command>$(command) ;
- }
- configure $(version) : $(options) ;
-}
-
-
-# 'configure' is a newer version of 'init'. The parameter 'command' is passed as
-# a part of the 'options' list. See the 'init' rule comment for more detailed
-# information.
-#
-rule configure ( version ? : options * )
-{
- switch $(version)
- {
- case "all" :
- if $(options)
- {
- import errors ;
- errors.error "MSVC toolset configuration: options should be"
- "empty when '$(version)' is specified." ;
- }
-
- # Configure (i.e. mark as used) all registered versions.
- local all-versions = [ $(.versions).all ] ;
- if ! $(all-versions)
- {
- if $(.debug-configuration)
- {
- ECHO "notice: [msvc-cfg] Asked to configure all registered"
- "msvc toolset versions when there are none currently"
- "registered." ;
- }
- }
- else
- {
- for local v in $(all-versions)
- {
- # Note that there is no need to skip already configured
- # versions here as this will request configure-really rule
- # to configure the version using default options which will
- # in turn cause it to simply do nothing in case the version
- # has already been configured.
- configure-really $(v) ;
- }
- }
-
- case "default" :
- configure-really : $(options) ;
-
- case * :
- configure-really $(version) : $(options) ;
- }
-}
-
-
-# Sets up flag definitions dependent on the compiler version used.
-# - 'version' is the version of compiler in N.M format.
-# - 'conditions' is the property set to be used as flag conditions.
-# - 'toolset' is the toolset for which flag settings are to be defined.
-# This makes the rule reusable for other msvc-option-compatible compilers.
-#
-rule configure-version-specific ( toolset : version : conditions )
-{
- toolset.push-checking-for-flags-module unchecked ;
- # Starting with versions 7.0, the msvc compiler have the /Zc:forScope and
- # /Zc:wchar_t options that improve C++ standard conformance, but those
- # options are off by default. If we are sure that the msvc version is at
- # 7.*, add those options explicitly. We can be sure either if user specified
- # version 7.* explicitly or if we auto-detected the version ourselves.
- if ! [ MATCH ^(6\\.) : $(version) ]
- {
- toolset.flags $(toolset).compile CFLAGS $(conditions) : /Zc:forScope /Zc:wchar_t ;
- toolset.flags $(toolset).compile.c++ C++FLAGS $(conditions) : /wd4675 ;
-
- # Explicitly disable the 'function is deprecated' warning. Some msvc
- # versions have a bug, causing them to emit the deprecation warning even
- # with /W0.
- toolset.flags $(toolset).compile CFLAGS $(conditions)/<warnings>off : /wd4996 ;
-
- if [ MATCH ^([78]\\.) : $(version) ]
- {
- # 64-bit compatibility warning deprecated since 9.0, see
- # http://msdn.microsoft.com/en-us/library/yt4xw8fh.aspx
- toolset.flags $(toolset).compile CFLAGS $(conditions)/<warnings>all : /Wp64 ;
- }
- }
-
- #
- # Processor-specific optimization.
- #
-
- if [ MATCH ^([67]) : $(version) ]
- {
- # 8.0 deprecates some of the options.
- toolset.flags $(toolset).compile CFLAGS $(conditions)/<optimization>speed $(conditions)/<optimization>space : /Ogiy /Gs ;
- toolset.flags $(toolset).compile CFLAGS $(conditions)/<optimization>speed : /Ot ;
- toolset.flags $(toolset).compile CFLAGS $(conditions)/<optimization>space : /Os ;
-
- toolset.flags $(toolset).compile CFLAGS $(conditions)/$(.cpu-arch-i386)/<instruction-set> : /GB ;
- toolset.flags $(toolset).compile CFLAGS $(conditions)/$(.cpu-arch-i386)/<instruction-set>i486 : /G4 ;
- toolset.flags $(toolset).compile CFLAGS $(conditions)/$(.cpu-arch-i386)/<instruction-set>$(.cpu-type-g5) : /G5 ;
- toolset.flags $(toolset).compile CFLAGS $(conditions)/$(.cpu-arch-i386)/<instruction-set>$(.cpu-type-g6) : /G6 ;
- toolset.flags $(toolset).compile CFLAGS $(conditions)/$(.cpu-arch-i386)/<instruction-set>$(.cpu-type-g7) : /G7 ;
-
- # Improve floating-point accuracy. Otherwise, some of C++ Boost's "math"
- # tests will fail.
- toolset.flags $(toolset).compile CFLAGS $(conditions) : /Op ;
-
- # 7.1 and below have single-threaded static RTL.
- toolset.flags $(toolset).compile CFLAGS $(conditions)/<runtime-debugging>off/<runtime-link>static/<threading>single : /ML ;
- toolset.flags $(toolset).compile CFLAGS $(conditions)/<runtime-debugging>on/<runtime-link>static/<threading>single : /MLd ;
- }
- else
- {
- # 8.0 and above adds some more options.
- toolset.flags $(toolset).compile CFLAGS $(conditions)/$(.cpu-arch-amd64)/<instruction-set> : /favor:blend ;
- toolset.flags $(toolset).compile CFLAGS $(conditions)/$(.cpu-arch-amd64)/<instruction-set>$(.cpu-type-em64t) : /favor:EM64T ;
- toolset.flags $(toolset).compile CFLAGS $(conditions)/$(.cpu-arch-amd64)/<instruction-set>$(.cpu-type-amd64) : /favor:AMD64 ;
-
- # 8.0 and above only has multi-threaded static RTL.
- toolset.flags $(toolset).compile CFLAGS $(conditions)/<runtime-debugging>off/<runtime-link>static/<threading>single : /MT ;
- toolset.flags $(toolset).compile CFLAGS $(conditions)/<runtime-debugging>on/<runtime-link>static/<threading>single : /MTd ;
-
- # Specify target machine type so the linker will not need to guess.
- toolset.flags $(toolset).link LINKFLAGS $(conditions)/$(.cpu-arch-amd64) : /MACHINE:X64 ;
- toolset.flags $(toolset).link LINKFLAGS $(conditions)/$(.cpu-arch-i386) : /MACHINE:X86 ;
- toolset.flags $(toolset).link LINKFLAGS $(conditions)/$(.cpu-arch-ia64) : /MACHINE:IA64 ;
- toolset.flags $(toolset).link LINKFLAGS $(conditions)/$(.cpu-arch-arm) : /MACHINE:ARM ;
-
- # Make sure that manifest will be generated even if there is no
- # dependencies to put there.
- toolset.flags $(toolset).link LINKFLAGS $(conditions) : /MANIFEST ;
- }
- toolset.pop-checking-for-flags-module ;
-}
-
-
-# Registers this toolset including all of its flags, features & generators. Does
-# nothing on repeated calls.
-#
-rule register-toolset ( )
-{
- if ! msvc in [ feature.values toolset ]
- {
- register-toolset-really ;
- }
-}
-
-
-# Declare action for creating static libraries. If library exists, remove it
-# before adding files. See
-# http://article.gmane.org/gmane.comp.lib.boost.build/4241 for rationale.
-if [ os.name ] in NT
-{
- # The 'DEL' command would issue a message to stdout if the file does not
- # exist, so need a check.
- actions archive
- {
- if exist "$(<[1])" DEL "$(<[1])"
- $(.LD) $(AROPTIONS) /out:"$(<[1])" @"@($(<[1]:W).rsp:E=$(.nl)"$(>)" $(.nl)$(LIBRARIES_MENTIONED_BY_FILE) $(.nl)"$(LIBRARY_OPTION)$(FINDLIBS_ST).lib" $(.nl)"$(LIBRARY_OPTION)$(FINDLIBS_SA).lib")"
- }
-}
-else
-{
- actions archive
- {
- $(.RM) "$(<[1])"
- $(.LD) $(AROPTIONS) /out:"$(<[1])" @"@($(<[1]:W).rsp:E=$(.nl)"$(>)" $(.nl)$(LIBRARIES_MENTIONED_BY_FILE) $(.nl)"$(LIBRARY_OPTION)$(FINDLIBS_ST).lib" $(.nl)"$(LIBRARY_OPTION)$(FINDLIBS_SA).lib")"
- }
-}
-
-
-# For the assembler the following options are turned on by default:
-#
-# -Zp4 align structures to 4 bytes
-# -Cp preserve case of user identifiers
-# -Cx preserve case in publics, externs
-#
-actions compile.asm
-{
- $(.ASM) -c -Zp4 -Cp -Cx -D$(DEFINES) $(ASMFLAGS) $(USER_ASMFLAGS) -Fo "$(<:W)" "$(>:W)"
-}
-
-
-rule compile.c ( targets + : sources * : properties * )
-{
- C++FLAGS on $(targets[1]) = ;
- get-rspline $(targets) : -TC ;
- compile-c-c++ $(<) : $(>) [ on $(<) return $(PCH_FILE) ] [ on $(<) return $(PCH_HEADER) ] ;
-}
-
-
-rule compile.c.preprocess ( targets + : sources * : properties * )
-{
- C++FLAGS on $(targets[1]) = ;
- get-rspline $(targets) : -TC ;
- preprocess-c-c++ $(<) : $(>) [ on $(<) return $(PCH_FILE) ] [ on $(<) return $(PCH_HEADER) ] ;
-}
-
-
-rule compile.c.pch ( targets + : sources * : properties * )
-{
- C++FLAGS on $(targets[1]) = ;
- get-rspline $(targets[1]) : -TC ;
- get-rspline $(targets[2]) : -TC ;
- local pch-source = [ on $(<) return $(PCH_SOURCE) ] ;
- if $(pch-source)
- {
- DEPENDS $(<) : $(pch-source) ;
- compile-c-c++-pch-s $(targets) : $(sources) $(pch-source) ;
- }
- else
- {
- compile-c-c++-pch $(targets) : $(sources) ;
- }
-}
-
-toolset.flags msvc YLOPTION : "-Yl" ;
-
-# Action for running the C/C++ compiler without using precompiled headers.
-#
-# WARNING: Synchronize any changes this in action with intel-win
-#
-# Notes regarding PDB generation, for when we use
-# <debug-symbols>on/<debug-store>database:
-#
-# 1. PDB_CFLAG is only set for <debug-symbols>on/<debug-store>database, ensuring
-# that the /Fd flag is dropped if PDB_CFLAG is empty.
-#
-# 2. When compiling executables's source files, PDB_NAME is set on a per-source
-# file basis by rule compile-c-c++. The linker will pull these into the
-# executable's PDB.
-#
-# 3. When compiling library's source files, PDB_NAME is updated to <libname>.pdb
-# for each source file by rule archive, as in this case compiler must be used
-# to create a single PDB for our library.
-#
-actions compile-c-c++ bind PDB_NAME
-{
- $(.CC) @"@($(<[1]:W).rsp:E="$(>[1]:W)" -Fo"$(<[1]:W)" $(PDB_CFLAG)"$(PDB_NAME)" -Yu"$(>[3]:D=)" -Fp"$(>[2]:W)" $(CC_RSPLINE))" $(.CC.FILTER)
-}
-
-actions preprocess-c-c++ bind PDB_NAME
-{
- $(.CC) @"@($(<[1]:W).rsp:E="$(>[1]:W)" -E $(PDB_CFLAG)"$(PDB_NAME)" -Yu"$(>[3]:D=)" -Fp"$(>[2]:W)" $(CC_RSPLINE))" >"$(<[1]:W)"
-}
-
-rule compile-c-c++ ( targets + : sources * )
-{
- DEPENDS $(<[1]) : [ on $(<[1]) return $(PCH_HEADER) ] ;
- DEPENDS $(<[1]) : [ on $(<[1]) return $(PCH_FILE) ] ;
- PDB_NAME on $(<) = $(<[1]:S=.pdb) ;
- LOCATE on $(<[1]:S=.pdb) = [ on $(<[1]) return $(LOCATE) ] ;
-}
-
-rule preprocess-c-c++ ( targets + : sources * )
-{
- DEPENDS $(<[1]) : [ on $(<[1]) return $(PCH_HEADER) ] ;
- DEPENDS $(<[1]) : [ on $(<[1]) return $(PCH_FILE) ] ;
- PDB_NAME on $(<) = $(<:S=.pdb) ;
- LOCATE on $(<[1]:S=.pdb) = [ on $(<[1]) return $(LOCATE) ] ;
-}
-
-# Action for running the C/C++ compiler using precompiled headers. In addition
-# to whatever else it needs to compile, this action also adds a temporary source
-# .cpp file used to compile the precompiled headers themselves.
-#
-# The global .escaped-double-quote variable is used to avoid messing up Emacs
-# syntax highlighting in the messy N-quoted code below.
-actions compile-c-c++-pch
-{
- $(.CC) @"@($(<[1]:W).rsp:E="$(>[2]:W)" -Fo"$(<[2]:W)" -Yc"$(>[1]:D=)" $(YLOPTION)"__bjam_pch_symbol_$(>[1]:D=)" -Fp"$(<[1]:W)" $(CC_RSPLINE))" "@($(<[1]:W).cpp:E=#include $(.escaped-double-quote)$(>[1]:D=)$(.escaped-double-quote)$(.nl))" $(.CC.FILTER)
-}
-
-
-# Action for running the C/C++ compiler using precompiled headers. An already
-# built source file for compiling the precompiled headers is expected to be
-# given as one of the source parameters.
-actions compile-c-c++-pch-s
-{
- $(.CC) @"@($(<[1]:W).rsp:E="$(>[2]:W)" -Fo"$(<[2]:W)" -Yc"$(>[1]:D=)" $(YLOPTION)"__bjam_pch_symbol_$(>[1]:D=)" -Fp"$(<[1]:W)" $(CC_RSPLINE))" $(.CC.FILTER)
-}
-
-
-rule compile.c++ ( targets + : sources * : properties * )
-{
- get-rspline $(targets) : -TP ;
- compile-c-c++ $(<) : $(>) [ on $(<) return $(PCH_FILE) ] [ on $(<) return $(PCH_HEADER) ] ;
-}
-
-rule compile.c++.preprocess ( targets + : sources * : properties * )
-{
- get-rspline $(targets) : -TP ;
- preprocess-c-c++ $(<) : $(>) [ on $(<) return $(PCH_FILE) ] [ on $(<) return $(PCH_HEADER) ] ;
-}
-
-
-rule compile.c++.pch ( targets + : sources * : properties * )
-{
- get-rspline $(targets[1]) : -TP ;
- get-rspline $(targets[2]) : -TP ;
- local pch-source = [ on $(<) return $(PCH_SOURCE) ] ;
- if $(pch-source)
- {
- DEPENDS $(<) : $(pch-source) ;
- compile-c-c++-pch-s $(targets) : $(sources) $(pch-source) ;
- }
- else
- {
- compile-c-c++-pch $(targets) : $(sources) ;
- }
-}
-
-
-# See midl.jam for details.
-#
-actions compile.idl
-{
- $(.IDL) /nologo @"@($(<[1]:W).rsp:E=$(.nl)"$(>:W)" $(.nl)-D$(DEFINES) $(.nl)"-I$(INCLUDES:W)" $(.nl)-U$(UNDEFS) $(.nl)$(MIDLFLAGS) $(.nl)/tlb "$(<[1]:W)" $(.nl)/h "$(<[2]:W)" $(.nl)/iid "$(<[3]:W)" $(.nl)/proxy "$(<[4]:W)" $(.nl)/dlldata "$(<[5]:W)")"
- $(.TOUCH_FILE) "$(<[4]:W)"
- $(.TOUCH_FILE) "$(<[5]:W)"
-}
-
-
-actions compile.mc
-{
- $(.MC) $(MCFLAGS) -h "$(<[1]:DW)" -r "$(<[2]:DW)" "$(>:W)"
-}
-
-
-actions compile.rc
-{
- $(.RC) -l 0x409 -U$(UNDEFS) -D$(DEFINES) -I"$(INCLUDES:W)" -fo "$(<:W)" "$(>:W)"
-}
-
-
-rule link ( targets + : sources * : properties * )
-{
- if <embed-manifest>on in $(properties)
- {
- msvc.manifest $(targets) : $(sources) : $(properties) ;
- }
-}
-
-rule link.dll ( targets + : sources * : properties * )
-{
- DEPENDS $(<) : [ on $(<) return $(DEF_FILE) ] ;
- if <embed-manifest>on in $(properties)
- {
- msvc.manifest.dll $(targets) : $(sources) : $(properties) ;
- }
-}
-
-# Incremental linking a DLL causes no end of problems: if the actual exports do
-# not change, the import .lib file is never updated. Therefore, the .lib is
-# always out-of-date and gets rebuilt every time. I am not sure that incremental
-# linking is such a great idea in general, but in this case I am sure we do not
-# want it.
-
-# Windows manifest is a new way to specify dependencies on managed DotNet
-# assemblies and Windows native DLLs. The manifests are embedded as resources
-# and are useful in any PE target (both DLL and EXE).
-
-if [ os.name ] in NT
-{
- actions link bind DEF_FILE LIBRARIES_MENTIONED_BY_FILE
- {
- $(.LD) $(LINKFLAGS) /out:"$(<[1]:W)" /LIBPATH:"$(LINKPATH:W)" $(OPTIONS) @"@($(<[1]:W).rsp:E=$(.nl)"$(>)" $(.nl)$(LIBRARIES_MENTIONED_BY_FILE) $(.nl)$(LIBRARIES) $(.nl)"$(LIBRARY_OPTION)$(FINDLIBS_ST).lib" $(.nl)"$(LIBRARY_OPTION)$(FINDLIBS_SA).lib")"
- if %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL%
- }
-
- actions manifest
- {
- if exist "$(<[1]).manifest" (
- $(.MT) -manifest "$(<[1]).manifest" "-outputresource:$(<[1]);1"
- )
- }
-
- actions link.dll bind DEF_FILE LIBRARIES_MENTIONED_BY_FILE
- {
- $(.LD) /DLL $(LINKFLAGS) /out:"$(<[1]:W)" /IMPLIB:"$(<[2]:W)" /LIBPATH:"$(LINKPATH:W)" /def:"$(DEF_FILE)" $(OPTIONS) @"@($(<[1]:W).rsp:E=$(.nl)"$(>)" $(.nl)$(LIBRARIES_MENTIONED_BY_FILE) $(.nl)$(LIBRARIES) $(.nl)"$(LIBRARY_OPTION)$(FINDLIBS_ST).lib" $(.nl)"$(LIBRARY_OPTION)$(FINDLIBS_SA).lib")"
- if %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL%
- }
-
- actions manifest.dll
- {
- if exist "$(<[1]).manifest" (
- $(.MT) -manifest "$(<[1]).manifest" "-outputresource:$(<[1]);2"
- )
- }
-}
-else
-{
- actions link bind DEF_FILE LIBRARIES_MENTIONED_BY_FILE
- {
- $(.LD) $(LINKFLAGS) /out:"$(<[1]:W)" /LIBPATH:"$(LINKPATH:W)" $(OPTIONS) @"@($(<[1]:W).rsp:E=$(.nl)"$(>)" $(.nl)$(LIBRARIES_MENTIONED_BY_FILE) $(.nl)$(LIBRARIES) $(.nl)"$(LIBRARY_OPTION)$(FINDLIBS_ST).lib" $(.nl)"$(LIBRARY_OPTION)$(FINDLIBS_SA).lib")"
- }
-
- actions manifest
- {
- if test -e "$(<[1]).manifest"; then
- $(.MT) -manifest "$(<[1]:W).manifest" "-outputresource:$(<[1]:W);1"
- fi
- }
-
- actions link.dll bind DEF_FILE LIBRARIES_MENTIONED_BY_FILE
- {
- $(.LD) /DLL $(LINKFLAGS) /out:"$(<[1]:W)" /IMPLIB:"$(<[2]:W)" /LIBPATH:"$(LINKPATH:W)" /def:"$(DEF_FILE)" $(OPTIONS) @"@($(<[1]:W).rsp:E=$(.nl)"$(>)" $(.nl)$(LIBRARIES_MENTIONED_BY_FILE) $(.nl)$(LIBRARIES) $(.nl)"$(LIBRARY_OPTION)$(FINDLIBS_ST).lib" $(.nl)"$(LIBRARY_OPTION)$(FINDLIBS_SA).lib")"
- }
-
- actions manifest.dll
- {
- if test -e "$(<[1]).manifest"; then
- $(.MT) -manifest "$(<[1]:W).manifest" "-outputresource:$(<[1]:W);2"
- fi
- }
-}
-
-# This rule sets up the pdb file that will be used when generating static
-# libraries and the debug-store option is database, so that the compiler puts
-# all the debug info into a single .pdb file named after the library.
-#
-# Poking at source targets this way is probably not clean, but it is the
-# easiest approach.
-#
-rule archive ( targets + : sources * : properties * )
-{
- PDB_NAME on $(>) = $(<[1]:S=.pdb) ;
- LOCATE on $(<[1]:S=.pdb) = [ on $(<[1]) return $(LOCATE) ] ;
-}
-
-
-################################################################################
-#
-# Classes.
-#
-################################################################################
-
-class msvc-pch-generator : pch-generator
-{
- import property-set ;
-
- rule run-pch ( project name ? : property-set : sources * )
- {
- # Searching for the header and source file in the sources.
- local pch-header ;
- local pch-source ;
- for local s in $(sources)
- {
- if [ type.is-derived [ $(s).type ] H ]
- {
- pch-header = $(s) ;
- }
- else if
- [ type.is-derived [ $(s).type ] CPP ] ||
- [ type.is-derived [ $(s).type ] C ]
- {
- pch-source = $(s) ;
- }
- }
-
- if ! $(pch-header)
- {
- import errors : user-error : errors.user-error ;
- errors.user-error "can not build pch without pch-header" ;
- }
-
- # If we do not have the PCH source - that is fine. We will just create a
- # temporary .cpp file in the action.
-
- local generated = [ generator.run $(project) $(name)
- : [ property-set.create
- # Passing of <pch-source> is a dirty trick, needed because
- # non-composing generators with multiple inputs are subtly
- # broken. For more detailed information see:
- # https://zigzag.cs.msu.su:7813/boost.build/ticket/111
- <pch-source>$(pch-source)
- [ $(property-set).raw ] ]
- : $(pch-header) ] ;
-
- local pch-file ;
- for local g in $(generated)
- {
- if [ type.is-derived [ $(g).type ] PCH ]
- {
- pch-file = $(g) ;
- }
- }
-
- return [ property-set.create <pch-header>$(pch-header)
- <pch-file>$(pch-file) ] $(generated) ;
- }
-}
-
-
-################################################################################
-#
-# Local rules.
-#
-################################################################################
-
-# Detects versions listed as '.known-versions' by checking registry information,
-# environment variables & default paths. Supports both native Windows and
-# Cygwin.
-#
-local rule auto-detect-toolset-versions ( )
-{
- if [ os.name ] in NT CYGWIN
- {
- # Get installation paths from the registry.
- for local i in $(.known-versions)
- {
- if $(.version-$(i)-reg)
- {
- local vc-path ;
- for local x in "" "Wow6432Node\\"
- {
- vc-path += [ W32_GETREG
- "HKEY_LOCAL_MACHINE\\SOFTWARE\\"$(x)"\\Microsoft\\"$(.version-$(i)-reg)
- : "ProductDir" ] ;
- }
-
- if $(vc-path)
- {
- vc-path = [ path.join [ path.make-NT $(vc-path[1]) ] "bin" ] ;
- register-configuration $(i) : [ path.native $(vc-path[1]) ] ;
- }
- }
- }
- }
-
- # Check environment and default installation paths.
- for local i in $(.known-versions)
- {
- if ! $(i) in [ $(.versions).all ]
- {
- register-configuration $(i) : [ default-path $(i) ] ;
- }
- }
-}
-
-
-# Worker rule for toolset version configuration. Takes an explicit version id or
-# nothing in case it should configure the default toolset version (the first
-# registered one or a new 'default' one in case no toolset versions have been
-# registered yet).
-#
-local rule configure-really ( version ? : options * )
-{
- local v = $(version) ;
-
- # Decide what the 'default' version is.
- if ! $(v)
- {
- # Take the first registered (i.e. auto-detected) version.
- version = [ $(.versions).all ] ;
- version = $(version[1]) ;
- v = $(version) ;
-
- # Note: 'version' can still be empty at this point if no versions have
- # been auto-detected.
- version ?= "default" ;
- }
-
- # Version alias -> real version number.
- if $(.version-alias-$(version))
- {
- version = $(.version-alias-$(version)) ;
- }
-
- # Check whether the selected configuration is already in use.
- if $(version) in [ $(.versions).used ]
- {
- # Allow multiple 'toolset.using' calls for the same configuration if the
- # identical sets of options are used.
- if $(options) && ( $(options) != [ $(.versions).get $(version) : options ] )
- {
- import errors ;
- errors.error "MSVC toolset configuration: Toolset version"
- "'$(version)' already configured." ;
- }
- }
- else
- {
- # Register a new configuration.
- $(.versions).register $(version) ;
-
- # Add user-supplied to auto-detected options.
- options = [ $(.versions).get $(version) : options ] $(options) ;
-
- # Mark the configuration as 'used'.
- $(.versions).use $(version) ;
-
- # Generate conditions and save them.
- local conditions = [ common.check-init-parameters msvc : version $(v) ]
- ;
-
- $(.versions).set $(version) : conditions : $(conditions) ;
-
- local command = [ feature.get-values <command> : $(options) ] ;
-
- # If version is specified, we try to search first in default paths, and
- # only then in PATH.
- command = [ common.get-invocation-command msvc : cl.exe : $(command) :
- [ default-paths $(version) ] : $(version) ] ;
-
- common.handle-options msvc : $(conditions) : $(command) : $(options) ;
-
- if ! $(version)
- {
- # Even if version is not explicitly specified, try to detect the
- # version from the path.
- # FIXME: We currently detect both Microsoft Visual Studio 9.0 and
- # 9.0express as 9.0 here.
- if [ MATCH "(Microsoft Visual Studio 12)" : $(command) ]
- {
- version = 12.0 ;
- }
- else if [ MATCH "(Microsoft Visual Studio 11)" : $(command) ]
- {
- version = 11.0 ;
- }
- else if [ MATCH "(Microsoft Visual Studio 10)" : $(command) ]
- {
- version = 10.0 ;
- }
- else if [ MATCH "(Microsoft Visual Studio 9)" : $(command) ]
- {
- version = 9.0 ;
- }
- else if [ MATCH "(Microsoft Visual Studio 8)" : $(command) ]
- {
- version = 8.0 ;
- }
- else if [ MATCH "(NET 2003[\/\\]VC7)" : $(command) ]
- {
- version = 7.1 ;
- }
- else if [ MATCH "(Microsoft Visual C\\+\\+ Toolkit 2003)" :
- $(command) ]
- {
- version = 7.1toolkit ;
- }
- else if [ MATCH "(.NET[\/\\]VC7)" : $(command) ]
- {
- version = 7.0 ;
- }
- else
- {
- version = 6.0 ;
- }
- }
-
- # Generate and register setup command.
-
- local below-8.0 = [ MATCH ^([67]\\.) : $(version) ] ;
-
- local cpu = i386 amd64 ia64 arm ;
- if $(below-8.0)
- {
- cpu = i386 ;
- }
-
- local setup-amd64 ;
- local setup-i386 ;
- local setup-ia64 ;
- local setup-arm ;
-
- if $(command)
- {
- # TODO: Note that if we specify a non-existant toolset version then
- # this rule may find and use a corresponding compiler executable
- # belonging to an incorrect toolset version. For example, if you
- # have only MSVC 7.1 installed, have its executable on the path and
- # specify you want Boost Build to use MSVC 9.0, then you want Boost
- # Build to report an error but this may cause it to silently use the
- # MSVC 7.1 compiler even though it thinks it is using the msvc-9.0
- # toolset version.
- command = [ common.get-absolute-tool-path $(command[-1]) ] ;
- }
-
- if $(command)
- {
- local parent = [ path.make $(command) ] ;
- parent = [ path.parent $(parent) ] ;
- parent = [ path.native $(parent) ] ;
-
- # Setup will be used if the command name has been specified. If
- # setup is not specified explicitly then a default setup script will
- # be used instead. Setup scripts may be global or arhitecture/
- # /platform/cpu specific. Setup options are used only in case of
- # global setup scripts.
-
- # Default setup scripts provided with different VC distributions:
- #
- # VC 7.1 had only the vcvars32.bat script specific to 32 bit i386
- # builds. It was located in the bin folder for the regular version
- # and in the root folder for the free VC 7.1 tools.
- #
- # Later 8.0 & 9.0 versions introduce separate platform specific
- # vcvars*.bat scripts (e.g. 32 bit, 64 bit AMD or 64 bit Itanium)
- # located in or under the bin folder. Most also include a global
- # vcvarsall.bat helper script located in the root folder which runs
- # one of the aforementioned vcvars*.bat scripts based on the options
- # passed to it. So far only the version coming with some PlatformSDK
- # distributions does not include this top level script but to
- # support those we need to fall back to using the worker scripts
- # directly in case the top level script can not be found.
-
- local global-setup = [ feature.get-values <setup> : $(options) ] ;
- global-setup = $(global-setup[1]) ;
- if ! $(below-8.0)
- {
- global-setup ?= [ locate-default-setup $(command) : $(parent) :
- vcvarsall.bat ] ;
- }
-
- local default-setup-amd64 = vcvarsx86_amd64.bat ;
- local default-setup-i386 = vcvars32.bat ;
- local default-setup-ia64 = vcvarsx86_ia64.bat ;
- local default-setup-arm = vcvarsx86_arm.bat ;
-
- # http://msdn2.microsoft.com/en-us/library/x4d2c09s(VS.80).aspx and
- # http://msdn2.microsoft.com/en-us/library/x4d2c09s(vs.90).aspx
- # mention an x86_IPF option, that seems to be a documentation bug
- # and x86_ia64 is the correct option.
- local default-global-setup-options-amd64 = x86_amd64 ;
- local default-global-setup-options-i386 = x86 ;
- local default-global-setup-options-ia64 = x86_ia64 ;
- local default-global-setup-options-arm = x86_arm ;
-
- # When using 64-bit Windows, and targeting 64-bit, it is possible to
- # use a native 64-bit compiler, selected by the "amd64" & "ia64"
- # parameters to vcvarsall.bat. There are two variables we can use --
- # PROCESSOR_ARCHITECTURE and PROCESSOR_IDENTIFIER. The first is
- # 'x86' when running 32-bit Windows, no matter which processor is
- # used, and 'AMD64' on 64-bit windows on x86 (either AMD64 or EM64T)
- # Windows.
- #
- if [ MATCH ^(AMD64) : [ os.environ PROCESSOR_ARCHITECTURE ] ]
- {
- default-global-setup-options-amd64 = amd64 ;
- }
- # TODO: The same 'native compiler usage' should be implemented for
- # the Itanium platform by using the "ia64" parameter. For this
- # though we need someone with access to this platform who can find
- # out how to correctly detect this case.
- else if $(somehow-detect-the-itanium-platform)
- {
- default-global-setup-options-ia64 = ia64 ;
- }
-
- local setup-prefix = "call " ;
- local setup-suffix = " >nul"$(.nl) ;
- if ! [ os.name ] in NT
- {
- setup-prefix = "cmd.exe /S /C call " ;
- setup-suffix = " \">nul\" \"&&\" " ;
- }
-
- for local c in $(cpu)
- {
- local setup-options ;
-
- setup-$(c) = [ feature.get-values <setup-$(c)> : $(options) ] ;
-
- if ! $(setup-$(c))-is-not-empty
- {
- if $(global-setup)-is-not-empty
- {
- setup-$(c) = $(global-setup) ;
-
- # If needed we can easily add using configuration flags
- # here for overriding which options get passed to the
- # global setup command for which target platform:
- # setup-options = [ feature.get-values <setup-options-$(c)> : $(options) ] ;
-
- setup-options ?= $(default-global-setup-options-$(c)) ;
- }
- else
- {
- setup-$(c) = [ locate-default-setup $(command) : $(parent) : $(default-setup-$(c)) ] ;
- }
- }
-
- # Cygwin to Windows path translation.
- setup-$(c) = "\""$(setup-$(c):W)"\"" ;
-
- # Append setup options to the setup name and add the final setup
- # prefix & suffix.
- setup-options ?= "" ;
- setup-$(c) = $(setup-prefix)$(setup-$(c):J=" ")" "$(setup-options:J=" ")$(setup-suffix) ;
- }
- }
-
- # Get tool names (if any) and finish setup.
-
- compiler = [ feature.get-values <compiler> : $(options) ] ;
- compiler ?= cl ;
-
- linker = [ feature.get-values <linker> : $(options) ] ;
- linker ?= link ;
-
- resource-compiler = [ feature.get-values <resource-compiler> : $(options) ] ;
- resource-compiler ?= rc ;
-
- # Turn on some options for i386 assembler
- # -coff generate COFF format object file (compatible with cl.exe output)
- local default-assembler-amd64 = ml64 ;
- local default-assembler-i386 = "ml -coff" ;
- local default-assembler-ia64 = ias ;
- local default-assembler-ia64 = armasm ;
-
- assembler = [ feature.get-values <assembler> : $(options) ] ;
-
- idl-compiler = [ feature.get-values <idl-compiler> : $(options) ] ;
- idl-compiler ?= midl ;
-
- mc-compiler = [ feature.get-values <mc-compiler> : $(options) ] ;
- mc-compiler ?= mc ;
-
- manifest-tool = [ feature.get-values <manifest-tool> : $(options) ] ;
- manifest-tool ?= mt ;
-
- local cc-filter = [ feature.get-values <compiler-filter> : $(options) ]
- ;
-
- for local c in $(cpu)
- {
- # Setup script is not required in some configurations.
- setup-$(c) ?= "" ;
-
- local cpu-conditions = $(conditions)/$(.cpu-arch-$(c)) ;
-
- if $(.debug-configuration)
- {
- for local cpu-condition in $(cpu-conditions)
- {
- ECHO "notice: [msvc-cfg] condition: '$(cpu-condition)', setup: '$(setup-$(c))'" ;
- }
- }
-
- local cpu-assembler = $(assembler) ;
- cpu-assembler ?= $(default-assembler-$(c)) ;
-
- toolset.flags msvc.compile .CC $(cpu-conditions) : $(setup-$(c))$(compiler) /Zm800 -nologo ;
- toolset.flags msvc.compile .RC $(cpu-conditions) : $(setup-$(c))$(resource-compiler) ;
- toolset.flags msvc.compile .ASM $(cpu-conditions) : $(setup-$(c))$(cpu-assembler) -nologo ;
- toolset.flags msvc.link .LD $(cpu-conditions) : $(setup-$(c))$(linker) /NOLOGO /INCREMENTAL:NO ;
- toolset.flags msvc.archive .LD $(cpu-conditions) : $(setup-$(c))$(linker) /lib /NOLOGO ;
- toolset.flags msvc.compile .IDL $(cpu-conditions) : $(setup-$(c))$(idl-compiler) ;
- toolset.flags msvc.compile .MC $(cpu-conditions) : $(setup-$(c))$(mc-compiler) ;
-
- toolset.flags msvc.link .MT $(cpu-conditions) : $(setup-$(c))$(manifest-tool) -nologo ;
-
- if $(cc-filter)
- {
- toolset.flags msvc .CC.FILTER $(cpu-conditions) : "|" $(cc-filter) ;
- }
- }
-
- # Set version-specific flags.
- configure-version-specific msvc : $(version) : $(conditions) ;
- }
-}
-
-
-# Returns the default installation path for the given version.
-#
-local rule default-path ( version )
-{
- # Use auto-detected path if possible.
- local path = [ feature.get-values <command> : [ $(.versions).get $(version)
- : options ] ] ;
-
- if $(path)
- {
- path = $(path:D) ;
- }
- else
- {
- # Check environment.
- if $(.version-$(version)-env)
- {
- local vc-path = [ os.environ $(.version-$(version)-env) ] ;
- if $(vc-path)
- {
- vc-path = [ path.make $(vc-path) ] ;
- vc-path = [ path.join $(vc-path) $(.version-$(version)-envpath) ] ;
- vc-path = [ path.native $(vc-path) ] ;
-
- path = $(vc-path) ;
- }
- }
-
- # Check default path.
- if ! $(path) && $(.version-$(version)-path)
- {
- path = [ path.native [ path.join $(.ProgramFiles) $(.version-$(version)-path) ] ] ;
- }
- }
-
- return $(path) ;
-}
-
-
-# Returns either the default installation path (if 'version' is not empty) or
-# list of all known default paths (if no version is given)
-#
-local rule default-paths ( version ? )
-{
- local possible-paths ;
-
- if $(version)
- {
- possible-paths += [ default-path $(version) ] ;
- }
- else
- {
- for local i in $(.known-versions)
- {
- possible-paths += [ default-path $(i) ] ;
- }
- }
-
- return $(possible-paths) ;
-}
-
-
-rule get-rspline ( target : lang-opt )
-{
- CC_RSPLINE on $(target) = [ on $(target) return $(lang-opt) -U$(UNDEFS)
- $(CFLAGS) $(C++FLAGS) $(OPTIONS) -c $(.nl)-D$(DEFINES)
- $(.nl)\"-I$(INCLUDES:W)\" ] ;
-}
-
-class msvc-linking-generator : linking-generator
-{
- # Calls the base version. If necessary, also create a target for the
- # manifest file.specifying source's name as the name of the created
- # target. As result, the PCH will be named whatever.hpp.gch, and not
- # whatever.gch.
- rule generated-targets ( sources + : property-set : project name ? )
- {
- local result = [ linking-generator.generated-targets $(sources)
- : $(property-set) : $(project) $(name) ] ;
-
- if $(result)
- {
- local name-main = [ $(result[0]).name ] ;
- local action = [ $(result[0]).action ] ;
-
- if [ $(property-set).get <debug-symbols> ] = "on"
- {
- # We force the exact name on PDB. The reason is tagging -- the
- # tag rule may reasonably special case some target types, like
- # SHARED_LIB. The tag rule will not catch PDBs, and it cannot
- # even easily figure out if a PDB is paired with a SHARED_LIB,
- # EXE or something else. Because PDBs always get the same name
- # as the main target, with .pdb as extension, just force it.
- local target = [ class.new file-target $(name-main:S=.pdb) exact
- : PDB : $(project) : $(action) ] ;
- local registered-target = [ virtual-target.register $(target) ]
- ;
- if $(target) != $(registered-target)
- {
- $(action).replace-targets $(target) : $(registered-target) ;
- }
- result += $(registered-target) ;
- }
-
- if [ $(property-set).get <embed-manifest> ] = "off"
- {
- # Manifest is an evil target. It has .manifest appened to the
- # name of the main target, including extension, e.g.
- # a.exe.manifest. We use the 'exact' name to achieve this
- # effect.
- local target = [ class.new file-target $(name-main).manifest
- exact : MANIFEST : $(project) : $(action) ] ;
- local registered-target = [ virtual-target.register $(target) ]
- ;
- if $(target) != $(registered-target)
- {
- $(action).replace-targets $(target) : $(registered-target) ;
- }
- result += $(registered-target) ;
- }
- }
- return $(result) ;
- }
-}
-
-
-# Unsafe worker rule for the register-toolset() rule. Must not be called
-# multiple times.
-#
-local rule register-toolset-really ( )
-{
- feature.extend toolset : msvc ;
-
- # Intel and msvc supposedly have link-compatible objects.
- feature.subfeature toolset msvc : vendor : intel : propagated optional ;
-
- # Inherit MIDL flags.
- toolset.inherit-flags msvc : midl ;
-
- # Inherit MC flags.
- toolset.inherit-flags msvc : mc ;
-
- # Dynamic runtime comes only in MT flavour.
- toolset.add-requirements
- <toolset>msvc,<runtime-link>shared:<threading>multi ;
-
- # Declare msvc toolset specific features.
- {
- feature.feature debug-store : object database : propagated ;
- feature.feature pch-source : : dependency free ;
- }
-
- # Declare generators.
- {
- # TODO: Is it possible to combine these? Make the generators
- # non-composing so that they do not convert each source into a separate
- # .rsp file.
- generators.register [ new msvc-linking-generator msvc.link :
- OBJ SEARCHED_LIB STATIC_LIB IMPORT_LIB : EXE : <toolset>msvc ] ;
- generators.register [ new msvc-linking-generator msvc.link.dll :
- OBJ SEARCHED_LIB STATIC_LIB IMPORT_LIB : SHARED_LIB IMPORT_LIB :
- <toolset>msvc ] ;
-
- generators.register-archiver msvc.archive : OBJ : STATIC_LIB : <toolset>msvc ;
- generators.register-c-compiler msvc.compile.c++ : CPP : OBJ : <toolset>msvc ;
- generators.register-c-compiler msvc.compile.c : C : OBJ : <toolset>msvc ;
- generators.register-c-compiler msvc.compile.c++.preprocess : CPP : PREPROCESSED_CPP : <toolset>msvc ;
- generators.register-c-compiler msvc.compile.c.preprocess : C : PREPROCESSED_C : <toolset>msvc ;
-
- # Using 'register-c-compiler' adds the build directory to INCLUDES.
- generators.register-c-compiler msvc.compile.rc : RC : OBJ(%_res) : <toolset>msvc ;
- generators.override msvc.compile.rc : rc.compile.resource ;
- generators.register-standard msvc.compile.asm : ASM : OBJ : <toolset>msvc ;
-
- generators.register-c-compiler msvc.compile.idl : IDL : MSTYPELIB H C(%_i) C(%_proxy) C(%_dlldata) : <toolset>msvc ;
- generators.override msvc.compile.idl : midl.compile.idl ;
-
- generators.register-standard msvc.compile.mc : MC : H RC : <toolset>msvc ;
- generators.override msvc.compile.mc : mc.compile ;
-
- # Note: the 'H' source type will catch both '.h' and '.hpp' headers as
- # the latter have their HPP type derived from H. The type of compilation
- # is determined entirely by the destination type.
- generators.register [ new msvc-pch-generator msvc.compile.c.pch : H : C_PCH OBJ : <pch>on <toolset>msvc ] ;
- generators.register [ new msvc-pch-generator msvc.compile.c++.pch : H : CPP_PCH OBJ : <pch>on <toolset>msvc ] ;
-
- generators.override msvc.compile.c.pch : pch.default-c-pch-generator ;
- generators.override msvc.compile.c++.pch : pch.default-cpp-pch-generator ;
- }
-
- toolset.flags msvc.compile PCH_FILE <pch>on : <pch-file> ;
- toolset.flags msvc.compile PCH_SOURCE <pch>on : <pch-source> ;
- toolset.flags msvc.compile PCH_HEADER <pch>on : <pch-header> ;
-
- #
- # Declare flags for compilation.
- #
-
- toolset.flags msvc.compile CFLAGS <optimization>speed : /O2 ;
- toolset.flags msvc.compile CFLAGS <optimization>space : /O1 ;
-
- toolset.flags msvc.compile CFLAGS $(.cpu-arch-ia64)/<instruction-set>$(.cpu-type-itanium) : /G1 ;
- toolset.flags msvc.compile CFLAGS $(.cpu-arch-ia64)/<instruction-set>$(.cpu-type-itanium2) : /G2 ;
-
- toolset.flags msvc.compile CFLAGS <debug-symbols>on/<debug-store>object : /Z7 ;
- toolset.flags msvc.compile CFLAGS <debug-symbols>on/<debug-store>database : /Zi ;
- toolset.flags msvc.compile CFLAGS <optimization>off : /Od ;
- toolset.flags msvc.compile CFLAGS <inlining>off : /Ob0 ;
- toolset.flags msvc.compile CFLAGS <inlining>on : /Ob1 ;
- toolset.flags msvc.compile CFLAGS <inlining>full : /Ob2 ;
-
- toolset.flags msvc.compile CFLAGS <warnings>on : /W3 ;
- toolset.flags msvc.compile CFLAGS <warnings>off : /W0 ;
- toolset.flags msvc.compile CFLAGS <warnings>all : /W4 ;
- toolset.flags msvc.compile CFLAGS <warnings-as-errors>on : /WX ;
-
- toolset.flags msvc.compile C++FLAGS <exception-handling>on/<asynch-exceptions>off/<extern-c-nothrow>off : /EHs ;
- toolset.flags msvc.compile C++FLAGS <exception-handling>on/<asynch-exceptions>off/<extern-c-nothrow>on : /EHsc ;
- toolset.flags msvc.compile C++FLAGS <exception-handling>on/<asynch-exceptions>on/<extern-c-nothrow>off : /EHa ;
- toolset.flags msvc.compile C++FLAGS <exception-handling>on/<asynch-exceptions>on/<extern-c-nothrow>on : /EHac ;
-
- # By default 8.0 enables rtti support while prior versions disabled it. We
- # simply enable or disable it explicitly so we do not have to depend on this
- # default behaviour.
- toolset.flags msvc.compile CFLAGS <rtti>on : /GR ;
- toolset.flags msvc.compile CFLAGS <rtti>off : /GR- ;
- toolset.flags msvc.compile CFLAGS <runtime-debugging>off/<runtime-link>shared : /MD ;
- toolset.flags msvc.compile CFLAGS <runtime-debugging>on/<runtime-link>shared : /MDd ;
-
- toolset.flags msvc.compile CFLAGS <runtime-debugging>off/<runtime-link>static/<threading>multi : /MT ;
- toolset.flags msvc.compile CFLAGS <runtime-debugging>on/<runtime-link>static/<threading>multi : /MTd ;
-
- toolset.flags msvc.compile OPTIONS <cflags> : ;
- toolset.flags msvc.compile.c++ OPTIONS <cxxflags> : ;
-
- toolset.flags msvc.compile PDB_CFLAG <debug-symbols>on/<debug-store>database : /Fd ;
-
- toolset.flags msvc.compile DEFINES <define> ;
- toolset.flags msvc.compile UNDEFS <undef> ;
- toolset.flags msvc.compile INCLUDES <include> ;
-
- # Declare flags for the assembler.
- toolset.flags msvc.compile.asm USER_ASMFLAGS <asmflags> ;
-
- toolset.flags msvc.compile.asm ASMFLAGS <debug-symbols>on : "/Zi /Zd" ;
-
- toolset.flags msvc.compile.asm ASMFLAGS <warnings>on : /W3 ;
- toolset.flags msvc.compile.asm ASMFLAGS <warnings>off : /W0 ;
- toolset.flags msvc.compile.asm ASMFLAGS <warnings>all : /W4 ;
- toolset.flags msvc.compile.asm ASMFLAGS <warnings-as-errors>on : /WX ;
-
- toolset.flags msvc.compile.asm DEFINES <define> ;
-
- # Declare flags for linking.
- {
- toolset.flags msvc.link PDB_LINKFLAG <debug-symbols>on/<debug-store>database : /PDB: ; # not used yet
- toolset.flags msvc.link LINKFLAGS <debug-symbols>on : /DEBUG ;
- toolset.flags msvc.link DEF_FILE <def-file> ;
-
- # The linker disables the default optimizations when using /DEBUG so we
- # have to enable them manually for release builds with debug symbols.
- toolset.flags msvc LINKFLAGS <debug-symbols>on/<runtime-debugging>off : /OPT:REF,ICF ;
-
- toolset.flags msvc LINKFLAGS <user-interface>console : /subsystem:console ;
- toolset.flags msvc LINKFLAGS <user-interface>gui : /subsystem:windows ;
- toolset.flags msvc LINKFLAGS <user-interface>wince : /subsystem:windowsce ;
- toolset.flags msvc LINKFLAGS <user-interface>native : /subsystem:native ;
- toolset.flags msvc LINKFLAGS <user-interface>auto : /subsystem:posix ;
-
- toolset.flags msvc.link OPTIONS <linkflags> ;
- toolset.flags msvc.link LINKPATH <library-path> ;
-
- toolset.flags msvc.link FINDLIBS_ST <find-static-library> ;
- toolset.flags msvc.link FINDLIBS_SA <find-shared-library> ;
- toolset.flags msvc.link LIBRARY_OPTION <toolset>msvc : "" : unchecked ;
- toolset.flags msvc.link LIBRARIES_MENTIONED_BY_FILE : <library-file> ;
- }
-
- toolset.flags msvc.archive AROPTIONS <archiveflags> ;
-}
-
-
-# Locates the requested setup script under the given folder and returns its full
-# path or nothing in case the script can not be found. In case multiple scripts
-# are found only the first one is returned.
-#
-# TODO: There used to exist a code comment for the msvc.init rule stating that
-# we do not correctly detect the location of the vcvars32.bat setup script for
-# the free VC7.1 tools in case user explicitly provides a path. This should be
-# tested or simply remove this whole comment in case this toolset version is no
-# longer important.
-#
-local rule locate-default-setup ( command : parent : setup-name )
-{
- local result = [ GLOB $(command) $(parent) : $(setup-name) ] ;
- if $(result[1])
- {
- return $(result[1]) ;
- }
-}
-
-
-# Validates given path, registers found configuration and prints debug
-# information about it.
-#
-local rule register-configuration ( version : path ? )
-{
- if $(path)
- {
- local command = [ GLOB $(path) : cl.exe ] ;
-
- if $(command)
- {
- if $(.debug-configuration)
- {
- ECHO notice: [msvc-cfg] msvc-$(version) detected, command:
- '$(command)' ;
- }
-
- $(.versions).register $(version) ;
- $(.versions).set $(version) : options : <command>$(command) ;
- }
- }
-}
-
-
-################################################################################
-#
-# Startup code executed when loading this module.
-#
-################################################################################
-
-if [ MATCH (--debug-configuration) : [ modules.peek : ARGV ] ]
-{
- .debug-configuration = true ;
-}
-
-# Miscellaneous constants.
-.RM = [ common.rm-command ] ;
-.nl = "
-" ;
-.ProgramFiles = [ path.make [ common.get-program-files-dir ] ] ;
-.escaped-double-quote = "\"" ;
-.TOUCH_FILE = [ common.file-touch-command ] ;
-
-# List of all registered configurations.
-.versions = [ new configurations ] ;
-
-# Supported CPU architectures.
-.cpu-arch-i386 =
- <architecture>/<address-model>
- <architecture>/<address-model>32
- <architecture>x86/<address-model>
- <architecture>x86/<address-model>32 ;
-
-.cpu-arch-amd64 =
- <architecture>/<address-model>64
- <architecture>x86/<address-model>64 ;
-
-.cpu-arch-ia64 =
- <architecture>ia64/<address-model>
- <architecture>ia64/<address-model>64 ;
-
-.cpu-arch-arm =
- <architecture>arm/<address-model>32 ;
-
-
-# Supported CPU types (only Itanium optimization options are supported from
-# VC++ 2005 on). See
-# http://msdn2.microsoft.com/en-us/library/h66s5s0e(vs.90).aspx for more
-# detailed information.
-.cpu-type-g5 = i586 pentium pentium-mmx ;
-.cpu-type-g6 = i686 pentiumpro pentium2 pentium3 pentium3m pentium-m k6
- k6-2 k6-3 winchip-c6 winchip2 c3 c3-2 ;
-.cpu-type-em64t = prescott nocona core2 corei7 corei7-avx core-avx-i
- conroe conroe-xe conroe-l allendale merom
- merom-xe kentsfield kentsfield-xe penryn wolfdale
- yorksfield nehalem sandy-bridge ivy-bridge haswell ;
-.cpu-type-amd64 = k8 opteron athlon64 athlon-fx k8-sse3 opteron-sse3
- athlon64-sse3 amdfam10 barcelona bdver1 bdver2 bdver3 btver1 btver2 ;
-.cpu-type-g7 = pentium4 pentium4m athlon athlon-tbird athlon-4 athlon-xp
- athlon-mp $(.cpu-type-em64t) $(.cpu-type-amd64) ;
-.cpu-type-itanium = itanium itanium1 merced ;
-.cpu-type-itanium2 = itanium2 mckinley ;
-
-
-# Known toolset versions, in order of preference.
-.known-versions = 12.0 11.0 10.0 10.0express 9.0 9.0express 8.0 8.0express 7.1
- 7.1toolkit 7.0 6.0 ;
-
-# Version aliases.
-.version-alias-6 = 6.0 ;
-.version-alias-6.5 = 6.0 ;
-.version-alias-7 = 7.0 ;
-.version-alias-8 = 8.0 ;
-.version-alias-9 = 9.0 ;
-.version-alias-10 = 10.0 ;
-.version-alias-11 = 11.0 ;
-.version-alias-12 = 12.0 ;
-
-# Names of registry keys containing the Visual C++ installation path (relative
-# to "HKEY_LOCAL_MACHINE\SOFTWARE\\Microsoft").
-.version-6.0-reg = "VisualStudio\\6.0\\Setup\\Microsoft Visual C++" ;
-.version-7.0-reg = "VisualStudio\\7.0\\Setup\\VC" ;
-.version-7.1-reg = "VisualStudio\\7.1\\Setup\\VC" ;
-.version-8.0-reg = "VisualStudio\\8.0\\Setup\\VC" ;
-.version-8.0express-reg = "VCExpress\\8.0\\Setup\\VC" ;
-.version-9.0-reg = "VisualStudio\\9.0\\Setup\\VC" ;
-.version-9.0express-reg = "VCExpress\\9.0\\Setup\\VC" ;
-.version-10.0-reg = "VisualStudio\\10.0\\Setup\\VC" ;
-.version-10.0express-reg = "VCExpress\\10.0\\Setup\\VC" ;
-.version-11.0-reg = "VisualStudio\\11.0\\Setup\\VC" ;
-.version-12.0-reg = "VisualStudio\\12.0\\Setup\\VC" ;
-
-# Visual C++ Toolkit 2003 does not store its installation path in the registry.
-# The environment variable 'VCToolkitInstallDir' and the default installation
-# path will be checked instead.
-.version-7.1toolkit-path = "Microsoft Visual C++ Toolkit 2003" "bin" ;
-.version-7.1toolkit-env = VCToolkitInstallDir ;
-
-# Path to the folder containing "cl.exe" relative to the value of the
-# corresponding environment variable.
-.version-7.1toolkit-envpath = "bin" ;
-
-
-# Auto-detect all the available msvc installations on the system.
-auto-detect-toolset-versions ;
-
-
-# And finally trigger the actual Boost Build toolset registration.
-register-toolset ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/notfile.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/notfile.jam b/ext/kenlm/jam-files/boost-build/tools/notfile.jam
deleted file mode 100644
index 7d0985b..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/notfile.jam
+++ /dev/null
@@ -1,65 +0,0 @@
-# Copyright (c) 2005 Vladimir Prus.
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-import "class" : new ;
-import generators ;
-import project ;
-import targets ;
-import toolset ;
-import type ;
-
-
-type.register NOTFILE_MAIN ;
-
-
-class notfile-generator : generator
-{
- rule __init__ ( * : * )
- {
- generator.__init__ $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8)
- : $(9) : $(10) : $(11) : $(12) : $(13) : $(14) : $(15) : $(16) :
- $(17) : $(18) : $(19) ;
- }
-
- rule run ( project name ? : property-set : sources * : multiple ? )
- {
- local action ;
- local action-name = [ $(property-set).get <action> ] ;
- local m = [ MATCH ^@(.*) : $(action-name) ] ;
- if $(m)
- {
- action = [ new action $(sources) : $(m[1]) : $(property-set) ] ;
- }
- else
- {
- action = [ new action $(sources) : notfile.run : $(property-set) ] ;
- }
- local t = [ new notfile-target $(name) : $(project) : $(action) ] ;
- return [ virtual-target.register $(t) ] ;
- }
-}
-
-
-generators.register [ new notfile-generator notfile.main : : NOTFILE_MAIN ] ;
-
-
-toolset.flags notfile.run ACTION : <action> ;
-
-
-actions run
-{
- $(ACTION)
-}
-
-
-rule notfile ( target-name : action + : sources * : requirements * :
- default-build * )
-{
- targets.create-typed-target NOTFILE_MAIN : [ project.current ] :
- $(target-name) : $(sources) : $(requirements) <action>$(action) :
- $(default-build) ;
-}
-
-IMPORT $(__name__) : notfile : : notfile ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/package.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/package.jam b/ext/kenlm/jam-files/boost-build/tools/package.jam
deleted file mode 100644
index 198c223..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/package.jam
+++ /dev/null
@@ -1,165 +0,0 @@
-# Copyright (c) 2005 Vladimir Prus.
-# Copyright 2006 Rene Rivera.
-#
-# Use, modification and distribution is subject to the Boost Software
-# License Version 1.0. (See accompanying file LICENSE_1_0.txt or
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# Provides mechanism for installing whole packages into a specific directory
-# structure. This is opposed to the 'install' rule, that installs a number of
-# targets to a single directory, and does not care about directory structure at
-# all.
-
-# Example usage:
-#
-# package.install boost : <properties>
-# : <binaries>
-# : <libraries>
-# : <headers>
-# ;
-#
-# This will install binaries, libraries and headers to the 'proper' location,
-# given by command line options --prefix, --exec-prefix, --bindir, --libdir and
-# --includedir.
-#
-# The rule is just a convenient wrapper, avoiding the need to define several
-# 'install' targets.
-#
-# The only install-related feature is <install-source-root>. It will apply to
-# headers only and if present, paths of headers relatively to source root will
-# be retained after installing. If it is not specified, then "." is assumed, so
-# relative paths in headers are always preserved.
-
-import "class" : new ;
-import option ;
-import project ;
-import feature ;
-import property ;
-import stage ;
-import targets ;
-import modules ;
-
-feature.feature install-default-prefix : : free incidental ;
-
-rule install ( name package-name ? : requirements * : binaries * : libraries * : headers * )
-{
- package-name ?= $(name) ;
- if [ MATCH --prefix=(.*) : [ modules.peek : ARGV ] ]
- {
- # If --prefix is explicitly specified on the command line,
- # then we need wipe away any settings of libdir/includir that
- # is specified via options in config files.
- option.set bindir : ;
- option.set libdir : ;
- option.set includedir : ;
- }
-
- # If <install-source-root> is not specified, all headers are installed to
- # prefix/include, no matter what their relative path is. Sometimes that is
- # what is needed.
- local install-source-root = [ property.select <install-source-root> :
- $(requirements) ] ;
- install-source-root = $(install-source-root:G=) ;
- requirements = [ property.change $(requirements) : <install-source-root> ] ;
-
- local install-header-subdir = [ property.select <install-header-subdir> :
- $(requirements) ] ;
- install-header-subdir = /$(install-header-subdir:G=) ;
- install-header-subdir ?= "" ;
- requirements = [ property.change $(requirements) : <install-header-subdir> ]
- ;
-
- # First, figure out all locations. Use the default if no prefix option
- # given.
- local prefix = [ get-prefix $(name) : $(requirements) ] ;
-
- # Architecture dependent files.
- local exec-locate = [ option.get exec-prefix : $(prefix) ] ;
-
- # Binaries.
- local bin-locate = [ option.get bindir : $(prefix)/bin ] ;
-
- # Object code libraries.
- local lib-locate = [ option.get libdir : $(prefix)/lib ] ;
-
- # Source header files.
- local include-locate = [ option.get includedir : $(prefix)/include ] ;
-
- stage.install $(name)-bin : $(binaries) : $(requirements)
- <location>$(bin-locate) ;
- alias $(name)-lib : $(name)-lib-shared $(name)-lib-static ;
-
- # Since the install location of shared libraries differs on universe
- # and cygwin, use target alternatives to make different targets.
- # We should have used indirection conditioanl requirements, but it's
- # awkward to pass bin-locate and lib-locate from there to another rule.
- alias $(name)-lib-shared : $(name)-lib-shared-universe ;
- alias $(name)-lib-shared : $(name)-lib-shared-cygwin : <target-os>cygwin ;
-
- # For shared libraries, we install both explicitly specified one and the
- # shared libraries that the installed executables depend on.
- stage.install $(name)-lib-shared-universe : $(binaries) $(libraries) : $(requirements)
- <location>$(lib-locate) <install-dependencies>on <install-type>SHARED_LIB ;
- stage.install $(name)-lib-shared-cygwin : $(binaries) $(libraries) : $(requirements)
- <location>$(bin-locate) <install-dependencies>on <install-type>SHARED_LIB ;
-
- # For static libraries, we do not care about executable dependencies, since
- # static libraries are already incorporated into them.
- stage.install $(name)-lib-static : $(libraries) : $(requirements)
- <location>$(lib-locate) <install-dependencies>on <install-type>STATIC_LIB ;
- stage.install $(name)-headers : $(headers) : $(requirements)
- <location>$(include-locate)$(install-header-subdir)
- <install-source-root>$(install-source-root) ;
- alias $(name) : $(name)-bin $(name)-lib $(name)-headers ;
-
- local c = [ project.current ] ;
- local project-module = [ $(c).project-module ] ;
- module $(project-module)
- {
- explicit $(1)-bin $(1)-lib $(1)-headers $(1) $(1)-lib-shared $(1)-lib-static
- $(1)-lib-shared-universe $(1)-lib-shared-cygwin ;
- }
-}
-
-rule install-data ( target-name : package-name : data * : requirements * )
-{
- package-name ?= target-name ;
- if [ MATCH --prefix=(.*) : [ modules.peek : ARGV ] ]
- {
- # If --prefix is explicitly specified on the command line,
- # then we need wipe away any settings of datarootdir
- option.set datarootdir : ;
- }
-
- local prefix = [ get-prefix $(package-name) : $(requirements) ] ;
- local datadir = [ option.get datarootdir : $(prefix)/share ] ;
-
- stage.install $(target-name)
- : $(data)
- : $(requirements) <location>$(datadir)/$(package-name)
- ;
-
- local c = [ project.current ] ;
- local project-module = [ $(c).project-module ] ;
- module $(project-module)
- {
- explicit $(1) ;
- }
-}
-
-local rule get-prefix ( package-name : requirements * )
-{
- local prefix = [ option.get prefix : [ property.select
- <install-default-prefix> : $(requirements) ] ] ;
- prefix = $(prefix:G=) ;
- requirements = [ property.change $(requirements) : <install-default-prefix>
- ] ;
- # Or some likely defaults if neither is given.
- if ! $(prefix)
- {
- if [ modules.peek : NT ] { prefix = C:\\$(package-name) ; }
- else if [ modules.peek : UNIX ] { prefix = /usr/local ; }
- }
- return $(prefix) ;
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/pathscale.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/pathscale.jam b/ext/kenlm/jam-files/boost-build/tools/pathscale.jam
deleted file mode 100644
index 94abcf1..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/pathscale.jam
+++ /dev/null
@@ -1,178 +0,0 @@
-# Copyright 2006 Noel Belcourt
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-import property ;
-import generators ;
-import toolset : flags ;
-import feature ;
-import type ;
-import os ;
-import common ;
-import fortran ;
-
-feature.extend toolset : pathscale ;
-toolset.inherit pathscale : unix ;
-generators.override pathscale.prebuilt : builtin.prebuilt ;
-generators.override pathscale.searched-lib-generator : searched-lib-generator ;
-
-# Documentation and toolchain description located
-# http://www.pathscale.com/docs.html
-
-rule init ( version ? : command * : options * )
-{
- command = [ common.get-invocation-command pathscale : pathCC : $(command)
- : /opt/ekopath/bin ] ;
-
- # Determine the version
- local command-string = $(command:J=" ") ;
- if $(command)
- {
- version ?= [ MATCH "^([0-9.]+)"
- : [ SHELL "$(command-string) -dumpversion" ] ] ;
- }
-
- local condition = [ common.check-init-parameters pathscale
- : version $(version) ] ;
-
- common.handle-options pathscale : $(condition) : $(command) : $(options) ;
-
- toolset.flags pathscale.compile.fortran90 OPTIONS $(condition) :
- [ feature.get-values <fflags> : $(options) ] : unchecked ;
-
- command_c = $(command_c[1--2]) $(command[-1]:B=pathcc) ;
-
- toolset.flags pathscale CONFIG_C_COMMAND $(condition) : $(command_c) ;
-
- # fortran support
- local f-command = [ common.get-invocation-command pathscale : pathf90 : $(command) ] ;
- local command_f = $(command_f[1--2]) $(f-command[-1]:B=pathf90) ;
- local command_f90 = $(command_f[1--2]) $(f-command[-1]:B=pathf90) ;
-
- toolset.flags pathscale CONFIG_F_COMMAND $(condition) : $(command_f) ;
- toolset.flags pathscale CONFIG_F90_COMMAND $(condition) : $(command_f90) ;
-
- # always link lib rt to resolve clock_gettime()
- flags pathscale.link FINDLIBS-SA : rt : unchecked ;
-
- switch [ os.name ]
- {
- case SOLARIS :
- toolset.flags pathscale.link RPATH_OPTION $(condition) : -Wl,-R, -Wl, : unchecked ;
-
- case * : # GNU
- toolset.flags pathscale.link RPATH_OPTION $(condition) : -Wl,-rpath= : unchecked ;
- }
-}
-
-# Declare generators
-generators.register-c-compiler pathscale.compile.c : C : OBJ : <toolset>pathscale ;
-generators.register-c-compiler pathscale.compile.c++ : CPP : OBJ : <toolset>pathscale ;
-generators.register-fortran-compiler pathscale.compile.fortran : FORTRAN : OBJ : <toolset>pathscale ;
-generators.register-fortran90-compiler pathscale.compile.fortran90 : FORTRAN90 : OBJ : <toolset>pathscale ;
-
-# Declare flags and actions for compilation
-flags pathscale.compile OPTIONS <optimization>off : -O0 ;
-flags pathscale.compile OPTIONS <optimization>speed : -O3 ;
-flags pathscale.compile OPTIONS <optimization>space : -Os ;
-
-flags pathscale.compile OPTIONS <inlining>off : -noinline ;
-flags pathscale.compile OPTIONS <inlining>on : -inline ;
-flags pathscale.compile OPTIONS <inlining>full : -inline ;
-
-flags pathscale.compile OPTIONS <warnings>off : -woffall ;
-flags pathscale.compile OPTIONS <warnings>on : -Wall ;
-flags pathscale.compile OPTIONS <warnings>all : -Wall -pedantic ;
-flags pathscale.compile OPTIONS <warnings-as-errors>on : -Werror ;
-
-flags pathscale.compile OPTIONS <debug-symbols>on : -ggdb ;
-flags pathscale.compile OPTIONS <profiling>on : -pg ;
-flags pathscale.compile OPTIONS <link>shared : -fPIC ;
-flags pathscale.compile OPTIONS <address-model>32 : -m32 ;
-flags pathscale.compile OPTIONS <address-model>64 : -m64 ;
-
-flags pathscale.compile USER_OPTIONS <cflags> ;
-flags pathscale.compile.c++ USER_OPTIONS <cxxflags> ;
-flags pathscale.compile DEFINES <define> ;
-flags pathscale.compile INCLUDES <include> ;
-
-flags pathscale.compile.fortran USER_OPTIONS <fflags> ;
-flags pathscale.compile.fortran90 USER_OPTIONS <fflags> ;
-
-actions compile.c
-{
- "$(CONFIG_C_COMMAND)" $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
-}
-
-actions compile.c++
-{
- "$(CONFIG_COMMAND)" $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
-}
-
-actions compile.fortran
-{
- "$(CONFIG_F_COMMAND)" $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
-}
-
-rule compile.fortran90 ( targets * : sources * : properties * )
-{
- # the space rule inserts spaces between targets and it's necessary
- SPACE on $(targets) = " " ;
- # Serialize execution of the compile.fortran90 action
- # F90 source must be compiled in a particular order so we
- # serialize the build as a parallel F90 compile might fail
- JAM_SEMAPHORE on $(targets) = <s>pathscale-f90-semaphore ;
-}
-
-actions compile.fortran90
-{
- "$(CONFIG_F90_COMMAND)" $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -module $(<[1]:D) -c -o "$(<)" "$(>)"
-}
-
-# Declare flags and actions for linking
-flags pathscale.link OPTIONS <debug-symbols>on : -ggdb -rdynamic ;
-# Strip the binary when no debugging is needed
-flags pathscale.link OPTIONS <debug-symbols>off : -g0 ;
-flags pathscale.link OPTIONS <profiling>on : -pg ;
-flags pathscale.link USER_OPTIONS <linkflags> ;
-flags pathscale.link LINKPATH <library-path> ;
-flags pathscale.link FINDLIBS-ST <find-static-library> ;
-flags pathscale.link FINDLIBS-SA <find-shared-library> ;
-flags pathscale.link FINDLIBS-SA <threading>multi : pthread ;
-flags pathscale.link LIBRARIES <library-file> ;
-flags pathscale.link LINK-RUNTIME <runtime-link>static : static ;
-flags pathscale.link LINK-RUNTIME <runtime-link>shared : dynamic ;
-flags pathscale.link RPATH <dll-path> ;
-# On gcc, there are separate options for dll path at runtime and
-# link time. On Solaris, there's only one: -R, so we have to use
-# it, even though it's bad idea.
-flags pathscale.link RPATH <xdll-path> ;
-
-rule link ( targets * : sources * : properties * )
-{
- SPACE on $(targets) = " " ;
-}
-
-actions link bind LIBRARIES
-{
- "$(CONFIG_COMMAND)" $(OPTIONS) $(USER_OPTIONS) -L"$(LINKPATH)" $(RPATH_OPTION:E=-Wl,-rpath=)"$(RPATH)" -o "$(<)" "$(>)" "$(LIBRARIES)" -l$(FINDLIBS-SA) -l$(FINDLIBS-ST)
-}
-
-# Slight mods for dlls
-rule link.dll ( targets * : sources * : properties * )
-{
- SPACE on $(targets) = " " ;
-}
-
-actions link.dll bind LIBRARIES
-{
- "$(CONFIG_COMMAND)" $(OPTIONS) $(USER_OPTIONS) -L"$(LINKPATH)" $(RPATH_OPTION:E=-Wl,-rpath=)"$(RPATH)" -o "$(<)" -Wl,-soname$(SPACE)-Wl,$(<[1]:D=) -shared "$(>)" "$(LIBRARIES)" -l$(FINDLIBS-SA) -l$(FINDLIBS-ST)
-}
-
-# Declare action for creating static libraries
-# "$(CONFIG_COMMAND)" -ar -o "$(<)" "$(>)"
-actions piecemeal archive
-{
- ar $(ARFLAGS) ru "$(<)" "$(>)"
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/pch.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/pch.jam b/ext/kenlm/jam-files/boost-build/tools/pch.jam
deleted file mode 100644
index 0c6e98f..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/pch.jam
+++ /dev/null
@@ -1,95 +0,0 @@
-# Copyright (c) 2005 Reece H. Dunn.
-# Copyright 2006 Ilya Sokolov
-#
-# Use, modification and distribution is subject to the Boost Software
-# License Version 1.0. (See accompanying file LICENSE_1_0.txt or
-# http://www.boost.org/LICENSE_1_0.txt)
-
-##### Using Precompiled Headers (Quick Guide) #####
-#
-# Make precompiled mypch.hpp:
-#
-# import pch ;
-#
-# cpp-pch mypch
-# : # sources
-# mypch.hpp
-# : # requiremnts
-# <toolset>msvc:<source>mypch.cpp
-# ;
-#
-# Add cpp-pch to sources:
-#
-# exe hello
-# : main.cpp hello.cpp mypch
-# ;
-
-import "class" : new ;
-import type ;
-import feature ;
-import generators ;
-
-type.register PCH : pch ;
-
-type.register C_PCH : : PCH ;
-type.register CPP_PCH : : PCH ;
-
-# Control precompiled header (PCH) generation.
-feature.feature pch :
- on
- off
- : propagated ;
-
-
-feature.feature pch-header : : free dependency ;
-feature.feature pch-file : : free dependency ;
-
-# Base PCH generator. The 'run' method has the logic to prevent this generator
-# from being run unless it's being used for a top-level PCH target.
-class pch-generator : generator
-{
- import property-set ;
-
- rule action-class ( )
- {
- return compile-action ;
- }
-
- rule run ( project name ? : property-set : sources + )
- {
- if ! $(name)
- {
- # Unless this generator is invoked as the top-most generator for a
- # main target, fail. This allows using 'H' type as input type for
- # this generator, while preventing Boost.Build to try this generator
- # when not explicitly asked for.
- #
- # One bad example is msvc, where pch generator produces both PCH
- # target and OBJ target, so if there's any header generated (like by
- # bison, or by msidl), we'd try to use pch generator to get OBJ from
- # that H, which is completely wrong. By restricting this generator
- # only to pch main target, such problem is solved.
- }
- else
- {
- local r = [ run-pch $(project) $(name)
- : [ $(property-set).add-raw <define>BOOST_BUILD_PCH_ENABLED ]
- : $(sources) ] ;
- return [ generators.add-usage-requirements $(r)
- : <define>BOOST_BUILD_PCH_ENABLED ] ;
- }
- }
-
- # This rule must be overridden by the derived classes.
- rule run-pch ( project name ? : property-set : sources + )
- {
- }
-}
-
-
-# NOTE: requirements are empty, default pch generator can be applied when
-# pch=off.
-generators.register
- [ new dummy-generator pch.default-c-pch-generator : : C_PCH ] ;
-generators.register
- [ new dummy-generator pch.default-cpp-pch-generator : : CPP_PCH ] ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/pgi.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/pgi.jam b/ext/kenlm/jam-files/boost-build/tools/pgi.jam
deleted file mode 100644
index 90e827d..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/pgi.jam
+++ /dev/null
@@ -1,147 +0,0 @@
-# Copyright Noel Belcourt 2007.
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-import property ;
-import generators ;
-import os ;
-import toolset : flags ;
-import feature ;
-import fortran ;
-import type ;
-import common ;
-import gcc ;
-
-feature.extend toolset : pgi ;
-toolset.inherit pgi : unix ;
-generators.override pgi.prebuilt : builtin.lib-generator ;
-generators.override pgi.searched-lib-generator : searched-lib-generator ;
-
-# Documentation and toolchain description located
-# http://www.pgroup.com/resources/docs.htm
-
-rule init ( version ? : command * : options * )
-{
- local condition = [ common.check-init-parameters pgi : version $(version) ] ;
-
- local l_command = [ common.get-invocation-command pgi : pgCC : $(command) ] ;
-
- common.handle-options pgi : $(condition) : $(l_command) : $(options) ;
-
- command_c = $(command_c[1--2]) $(l_command[-1]:B=pgcc) ;
-
- toolset.flags pgi CONFIG_C_COMMAND $(condition) : $(command_c) ;
-
- flags pgi.compile DEFINES $(condition) :
- [ feature.get-values <define> : $(options) ] : unchecked ;
-
- # IOV_MAX support
- flags pgi.compile DEFINES $(condition) : __need_IOV_MAX : unchecked ;
-
- # set link flags
- flags pgi.link FINDLIBS-ST : [
- feature.get-values <find-static-library> : $(options) ] : unchecked ;
-
- # always link lib rt to resolve clock_gettime()
- flags pgi.link FINDLIBS-SA : rt [
- feature.get-values <find-shared-library> : $(options) ] : unchecked ;
-
- gcc.init-link-flags pgi gnu $(condition) ;
-}
-
-# Declare generators
-generators.register-c-compiler pgi.compile.c : C : OBJ : <toolset>pgi ;
-generators.register-c-compiler pgi.compile.c++ : CPP : OBJ : <toolset>pgi ;
-generators.register-fortran-compiler pgi.compile.fortran : FORTRAN : OBJ : <toolset>pgi ;
-
-# Declare flags and actions for compilation
-flags pgi.compile OPTIONS : -Kieee ;
-flags pgi.compile OPTIONS <link>shared : -fpic -fPIC ;
-flags pgi.compile OPTIONS <debug-symbols>on : -gopt ;
-flags pgi.compile OPTIONS <profiling>on : -xprofile=tcov ;
-flags pgi.compile OPTIONS <optimization>speed : -fast -Mx,8,0x10000000 ;
-flags pgi.compile OPTIONS <optimization>space : -xO2 -xspace ;
-# flags pgi.compile OPTIONS <threading>multi : -mt ;
-
-flags pgi.compile OPTIONS <warnings>off : -Minform=severe ;
-flags pgi.compile OPTIONS <warnings>on : -Minform=warn ;
-
-flags pgi.compile.c++ OPTIONS <inlining>off : -INLINE:none ;
-
-flags pgi.compile OPTIONS <cflags> ;
-flags pgi.compile.c++ OPTIONS <cxxflags> ;
-flags pgi.compile DEFINES <define> ;
-flags pgi.compile INCLUDES <include> ;
-
-flags pgi.compile.fortran OPTIONS <fflags> ;
-
-actions compile.c
-{
- "$(CONFIG_C_COMMAND)" $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
-}
-
-actions compile.c++
-{
- "$(CONFIG_COMMAND)" $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
-}
-
-actions compile.fortran
-{
- "$(CONFIG_F_COMMAND)" $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
-}
-
-# Declare flags and actions for linking
-flags pgi.link OPTIONS <debug-symbols>on : -gopt ;
-# Strip the binary when no debugging is needed
-flags pgi.link OPTIONS <debug-symbols>off : -s ;
-flags pgi.link OPTIONS <profiling>on : -xprofile=tcov ;
-flags pgi.link OPTIONS <linkflags> ;
-flags pgi.link OPTIONS <link>shared : -fpic -fPIC ;
-flags pgi.link LINKPATH <library-path> ;
-flags pgi.link FINDLIBS-ST <find-static-library> ;
-flags pgi.link FINDLIBS-SA <find-shared-library> ;
-flags pgi.link FINDLIBS-SA <threading>multi : pthread rt ;
-flags pgi.link LIBRARIES <library-file> ;
-flags pgi.link LINK-RUNTIME <runtime-link>static : static ;
-flags pgi.link LINK-RUNTIME <runtime-link>shared : dynamic ;
-flags pgi.link RPATH <dll-path> ;
-
-# On gcc, there are separate options for dll path at runtime and
-# link time. On Solaris, there's only one: -R, so we have to use
-# it, even though it's bad idea.
-flags pgi.link RPATH <xdll-path> ;
-
-rule link ( targets * : sources * : properties * )
-{
- SPACE on $(targets) = " " ;
-}
-
-# reddish can only link statically and, somehow, the presence of -Bdynamic on the link line
-# marks the executable as a dynamically linked exec even though no dynamic libraries are supplied.
-# Yod on redstorm refuses to load an executable that is dynamically linked.
-# removing the dynamic link options should get us where we need to be on redstorm.
-# "$(CONFIG_COMMAND)" $(OPTIONS) -L"$(LINKPATH)" -R"$(RPATH)" -o "$(<)" "$(>)" "$(LIBRARIES)" -Bdynamic -l$(FINDLIBS-SA) -Bstatic -l$(FINDLIBS-ST) -B$(LINK-RUNTIME)
-actions link bind LIBRARIES
-{
- "$(CONFIG_COMMAND)" $(OPTIONS) -L"$(LINKPATH)" -R"$(RPATH)" -o "$(<)" "$(>)" "$(LIBRARIES)" -Bstatic -l$(FINDLIBS-ST) -Bdynamic -l$(FINDLIBS-SA) -B$(LINK-RUNTIME)
-}
-
-# Slight mods for dlls
-rule link.dll ( targets * : sources * : properties * )
-{
- SPACE on $(targets) = " " ;
-}
-
-# "$(CONFIG_COMMAND)" $(OPTIONS) -L"$(LINKPATH)" -R"$(RPATH)" -o "$(<)" -h$(<[1]:D=) -G "$(>)" "$(LIBRARIES)" -Bdynamic -l$(FINDLIBS-SA) -Bstatic -l$(FINDLIBS-ST) -B$(LINK-RUNTIME)
-
-actions link.dll bind LIBRARIES
-{
- "$(CONFIG_COMMAND)" $(OPTIONS) -shared -L"$(LINKPATH)" -R"$(RPATH)" -o "$(<)" "$(>)" -Wl,-h -Wl,$(<[1]:D=) "$(LIBRARIES)" -Bdynamic -l$(FINDLIBS-SA) -Bstatic -l$(FINDLIBS-ST) -B$(LINK-RUNTIME)
-}
-
-actions updated together piecemeal pgi.archive
-{
- ar -rc$(ARFLAGS:E=) "$(<)" "$(>)"
-}
-
[19/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/make1.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/make1.c b/ext/kenlm/jam-files/engine/make1.c
deleted file mode 100644
index 71eee12..0000000
--- a/ext/kenlm/jam-files/engine/make1.c
+++ /dev/null
@@ -1,1283 +0,0 @@
-/*
- * Copyright 1993-2002 Christopher Seiwald and Perforce Software, Inc.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * make1.c - execute commands to bring targets up to date
- *
- * This module contains make1(), the entry point called by make() to recursively
- * descend the dependency graph executing update actions as marked by make0().
- *
- * External routines:
- * make1() - execute commands to update a TARGET and all of its dependencies
- *
- * Internal routines, the recursive/asynchronous command executors:
- * make1a() - recursively schedules dependency builds and then goes to
- * MAKE1B
- * make1b() - if nothing is blocking this target's build, proceed to
- * MAKE1C
- * make1c() - launch target's next command, or go to parents' MAKE1B
- * if none
- * make1c_closure() - handle command execution completion and go to MAKE1C
- *
- * Internal support routines:
- * make1cmds() - turn ACTIONS into CMDs, grouping, splitting, etc.
- * make1list() - turn a list of targets into a LIST, for $(<) and $(>)
- * make1settings() - for vars with bound values, build up replacement lists
- * make1bind() - bind targets that weren't bound in dependency analysis
- */
-
-#include "jam.h"
-#include "make.h"
-
-#include "command.h"
-#include "compile.h"
-#include "execcmd.h"
-#include "headers.h"
-#include "lists.h"
-#include "object.h"
-#include "output.h"
-#include "parse.h"
-#include "rules.h"
-#include "search.h"
-#include "variable.h"
-
-#include <assert.h>
-#include <stdlib.h>
-
-#if !defined( NT ) || defined( __GNUC__ )
- #include <unistd.h> /* for unlink */
-#endif
-
-static CMD * make1cmds ( TARGET * );
-static LIST * make1list ( LIST *, TARGETS *, int flags );
-static SETTINGS * make1settings ( struct module_t *, LIST * vars );
-static void make1bind ( TARGET * );
-static TARGET * make1findcycle ( TARGET * );
-static void make1breakcycle( TARGET *, TARGET * cycle_root );
-
-/* Ugly static - it is too hard to carry it through the callbacks. */
-
-static struct
-{
- int failed;
- int skipped;
- int total;
- int made;
-} counts[ 1 ];
-
-/* Target state. */
-#define T_STATE_MAKE1A 0 /* make1a() should be called */
-#define T_STATE_MAKE1B 1 /* make1b() should be called */
-#define T_STATE_MAKE1C 2 /* make1c() should be called */
-
-typedef struct _state state;
-struct _state
-{
- state * prev; /* previous state on stack */
- TARGET * t; /* current target */
- TARGET * parent; /* parent argument necessary for MAKE1A */
- int curstate; /* current state */
-};
-
-static void make1a( state * const );
-static void make1b( state * const );
-static void make1c( state const * const );
-
-static void make1c_closure( void * const closure, int status,
- timing_info const * const, char const * const cmd_stdout,
- char const * const cmd_stderr, int const cmd_exit_reason );
-
-typedef struct _stack
-{
- state * stack;
-} stack;
-
-static stack state_stack = { NULL };
-
-static state * state_freelist = NULL;
-
-/* Currently running command counter. */
-static int cmdsrunning;
-
-
-static state * alloc_state()
-{
- if ( state_freelist )
- {
- state * const pState = state_freelist;
- state_freelist = pState->prev;
- memset( pState, 0, sizeof( state ) );
- return pState;
- }
- return (state *)BJAM_MALLOC( sizeof( state ) );
-}
-
-
-static void free_state( state * const pState )
-{
- pState->prev = state_freelist;
- state_freelist = pState;
-}
-
-
-static void clear_state_freelist()
-{
- while ( state_freelist )
- {
- state * const pState = state_freelist;
- state_freelist = state_freelist->prev;
- BJAM_FREE( pState );
- }
-}
-
-
-static state * current_state( stack * const pStack )
-{
- return pStack->stack;
-}
-
-
-static void pop_state( stack * const pStack )
-{
- if ( pStack->stack )
- {
- state * const pState = pStack->stack->prev;
- free_state( pStack->stack );
- pStack->stack = pState;
- }
-}
-
-
-static state * push_state( stack * const pStack, TARGET * const t,
- TARGET * const parent, int const curstate )
-{
- state * const pState = alloc_state();
- pState->t = t;
- pState->parent = parent;
- pState->prev = pStack->stack;
- pState->curstate = curstate;
- return pStack->stack = pState;
-}
-
-
-/*
- * Pushes a stack onto another stack, effectively reversing the order.
- */
-
-static void push_stack_on_stack( stack * const pDest, stack * const pSrc )
-{
- while ( pSrc->stack )
- {
- state * const pState = pSrc->stack;
- pSrc->stack = pState->prev;
- pState->prev = pDest->stack;
- pDest->stack = pState;
- }
-}
-
-
-/*
- * make1() - execute commands to update a list of targets and all of their dependencies
- */
-
-static int intr = 0;
-static int quit = 0;
-
-int make1( LIST * targets )
-{
- state * pState;
- int status = 0;
-
- memset( (char *)counts, 0, sizeof( *counts ) );
-
- {
- LISTITER iter, end;
- stack temp_stack = { NULL };
- for ( iter = list_begin( targets ), end = list_end( targets );
- iter != end; iter = list_next( iter ) )
- push_state( &temp_stack, bindtarget( list_item( iter ) ), NULL, T_STATE_MAKE1A );
- push_stack_on_stack( &state_stack, &temp_stack );
- }
-
- /* Clear any state left over from the past */
- quit = 0;
-
- /* Recursively make the target and its dependencies. */
-
- while ( 1 )
- {
- while ( ( pState = current_state( &state_stack ) ) )
- {
- if ( quit )
- pop_state( &state_stack );
-
- switch ( pState->curstate )
- {
- case T_STATE_MAKE1A: make1a( pState ); break;
- case T_STATE_MAKE1B: make1b( pState ); break;
- case T_STATE_MAKE1C: make1c( pState ); break;
- default:
- assert( !"make1(): Invalid state detected." );
- }
- }
- if ( !cmdsrunning )
- break;
- /* Wait for outstanding commands to finish running. */
- exec_wait();
- }
-
- clear_state_freelist();
-
- /* Talk about it. */
- if ( counts->failed )
- printf( "...failed updating %d target%s...\n", counts->failed,
- counts->failed > 1 ? "s" : "" );
- if ( DEBUG_MAKE && counts->skipped )
- printf( "...skipped %d target%s...\n", counts->skipped,
- counts->skipped > 1 ? "s" : "" );
- if ( DEBUG_MAKE && counts->made )
- printf( "...updated %d target%s...\n", counts->made,
- counts->made > 1 ? "s" : "" );
-
- /* If we were interrupted, exit now that all child processes
- have finished. */
- if ( intr )
- exit( 1 );
-
- {
- LISTITER iter, end;
- for ( iter = list_begin( targets ), end = list_end( targets );
- iter != end; iter = list_next( iter ) )
- {
- /* Check that the target was updated and that the
- update succeeded. */
- TARGET * t = bindtarget( list_item( iter ) );
- if (t->progress == T_MAKE_DONE)
- {
- if (t->status != EXEC_CMD_OK)
- status = 1;
- }
- else if ( ! ( t->progress == T_MAKE_NOEXEC_DONE && globs.noexec ) )
- {
- status = 1;
- }
- }
- }
- return status;
-}
-
-
-/*
- * make1a() - recursively schedules dependency builds and then goes to MAKE1B
- *
- * Called to start processing a specified target. Does nothing if the target is
- * already being processed or otherwise starts processing all of its
- * dependencies.
- */
-
-static void make1a( state * const pState )
-{
- TARGET * t = pState->t;
- TARGET * const scc_root = target_scc( t );
-
- if ( !pState->parent || target_scc( pState->parent ) != scc_root )
- pState->t = t = scc_root;
-
- /* If the parent is the first to try to build this target or this target is
- * in the MAKE1C quagmire, arrange for the parent to be notified when this
- * target has been built.
- */
- if ( pState->parent && t->progress <= T_MAKE_RUNNING )
- {
- TARGET * const parent_scc = target_scc( pState->parent );
- if ( t != parent_scc )
- {
- t->parents = targetentry( t->parents, parent_scc );
- ++parent_scc->asynccnt;
- }
- }
-
- /* If the target has been previously updated with -n in effect, and we are
- * now ignoring -n, update it for real. E.g. if the UPDATE_NOW rule was
- * called for it twice - first with the -n option and then without.
- */
- if ( !globs.noexec && t->progress == T_MAKE_NOEXEC_DONE )
- t->progress = T_MAKE_INIT;
-
- /* If this target is already being processed then do nothing. There is no
- * need to start processing the same target all over again.
- */
- if ( t->progress != T_MAKE_INIT )
- {
- pop_state( &state_stack );
- return;
- }
-
- /* Guard against circular dependencies. */
- t->progress = T_MAKE_ONSTACK;
-
- /* 'asynccnt' counts the dependencies preventing this target from proceeding
- * to MAKE1C for actual building. We start off with a count of 1 to prevent
- * anything from happening until we can notify all dependencies that they
- * are needed. This 1 is then accounted for when we enter MAKE1B ourselves,
- * below. Without this if a dependency gets built before we finish
- * processing all of our other dependencies our build might be triggerred
- * prematurely.
- */
- t->asynccnt = 1;
-
- /* Push dependency build requests (to be executed in the natural order). */
- {
- stack temp_stack = { NULL };
- TARGETS * c;
- for ( c = t->depends; c && !quit; c = c->next )
- push_state( &temp_stack, c->target, t, T_STATE_MAKE1A );
- push_stack_on_stack( &state_stack, &temp_stack );
- }
-
- t->progress = T_MAKE_ACTIVE;
-
- /* Once all of our dependencies have started getting processed we can move
- * onto MAKE1B.
- */
- /* Implementation note:
- * In theory this would be done by popping this state before pushing
- * dependency target build requests but as a slight optimization we simply
- * modify our current state and leave it on the stack instead.
- */
- pState->curstate = T_STATE_MAKE1B;
-}
-
-
-/*
- * make1b() - if nothing is blocking this target's build, proceed to MAKE1C
- *
- * Called after something stops blocking this target's build, e.g. that all of
- * its dependencies have started being processed, one of its dependencies has
- * been built or a semaphore this target has been waiting for is free again.
- */
-
-static void make1b( state * const pState )
-{
- TARGET * const t = pState->t;
- TARGET * failed = 0;
- char const * failed_name = "dependencies";
-
- /* If any dependencies are still outstanding, wait until they signal their
- * completion by pushing this same state for their parent targets.
- */
- if ( --t->asynccnt )
- {
- pop_state( &state_stack );
- return;
- }
-
- /* Try to aquire a semaphore. If it is locked, wait until the target that
- * locked it is built and signals completition.
- */
-#ifdef OPT_SEMAPHORE
- if ( t->semaphore && t->semaphore->asynccnt )
- {
- /* Append 't' to the list of targets waiting on semaphore. */
- t->semaphore->parents = targetentry( t->semaphore->parents, t );
- t->asynccnt++;
-
- if ( DEBUG_EXECCMD )
- printf( "SEM: %s is busy, delaying launch of %s\n",
- object_str( t->semaphore->name ), object_str( t->name ) );
- pop_state( &state_stack );
- return;
- }
-#endif
-
- /* Now ready to build target 't', if dependencies built OK. */
-
- /* Collect status from dependencies. If -n was passed then act as though all
- * dependencies built correctly (the only way they can fail is if UPDATE_NOW
- * was called). If the dependencies can not be found or we got an interrupt,
- * we can not get here.
- */
- if ( !globs.noexec )
- {
- TARGETS * c;
- for ( c = t->depends; c; c = c->next )
- if ( c->target->status > t->status && !( c->target->flags &
- T_FLAG_NOCARE ) )
- {
- failed = c->target;
- t->status = c->target->status;
- }
- }
-
- /* If an internal header node failed to build, we want to output the target
- * that it failed on.
- */
- if ( failed )
- failed_name = failed->flags & T_FLAG_INTERNAL
- ? failed->failed
- : object_str( failed->name );
- t->failed = failed_name;
-
- /* If actions for building any of the dependencies have failed, bail.
- * Otherwise, execute all actions to make the current target.
- */
- if ( ( t->status == EXEC_CMD_FAIL ) && t->actions )
- {
- ++counts->skipped;
- if ( ( t->flags & ( T_FLAG_RMOLD | T_FLAG_NOTFILE ) ) == T_FLAG_RMOLD )
- {
- if ( !unlink( object_str( t->boundname ) ) )
- printf( "...removing outdated %s\n", object_str( t->boundname )
- );
- }
- else
- printf( "...skipped %s for lack of %s...\n", object_str( t->name ),
- failed_name );
- }
-
- if ( t->status == EXEC_CMD_OK )
- switch ( t->fate )
- {
- case T_FATE_STABLE:
- case T_FATE_NEWER:
- break;
-
- case T_FATE_CANTFIND:
- case T_FATE_CANTMAKE:
- t->status = EXEC_CMD_FAIL;
- break;
-
- case T_FATE_ISTMP:
- if ( DEBUG_MAKE )
- printf( "...using %s...\n", object_str( t->name ) );
- break;
-
- case T_FATE_TOUCHED:
- case T_FATE_MISSING:
- case T_FATE_NEEDTMP:
- case T_FATE_OUTDATED:
- case T_FATE_UPDATE:
- case T_FATE_REBUILD:
- /* Prepare commands for executing actions scheduled for this target.
- * Commands have their embedded variables automatically expanded,
- * including making use of any "on target" variables.
- */
- if ( t->actions )
- {
- ++counts->total;
- if ( DEBUG_MAKE && !( counts->total % 100 ) )
- printf( "...on %dth target...\n", counts->total );
-
- t->cmds = (char *)make1cmds( t );
- /* Update the target's "progress" so MAKE1C processing counts it
- * among its successes/failures.
- */
- t->progress = T_MAKE_RUNNING;
- }
- break;
-
- /* All valid fates should have been accounted for by now. */
- default:
- printf( "ERROR: %s has bad fate %d", object_str( t->name ),
- t->fate );
- abort();
- }
-
-#ifdef OPT_SEMAPHORE
- /* If there is a semaphore, indicate that it is in use. */
- if ( t->semaphore )
- {
- ++t->semaphore->asynccnt;
- if ( DEBUG_EXECCMD )
- printf( "SEM: %s now used by %s\n", object_str( t->semaphore->name
- ), object_str( t->name ) );
- }
-#endif
-
- /* Proceed to MAKE1C to begin executing the chain of commands prepared for
- * building the target. If we are not going to build the target (e.g. due to
- * dependency failures or no commands needing to be run) the chain will be
- * empty and MAKE1C processing will directly signal the target's completion.
- */
- /* Implementation note:
- * Morfing the current state on the stack instead of popping it and
- * pushing a new one is a slight optimization with no side-effects since we
- * pushed no other states while processing this one.
- */
- pState->curstate = T_STATE_MAKE1C;
-}
-
-
-/*
- * make1c() - launch target's next command, or go to parents' MAKE1B if none
- *
- * If there are (more) commands to run to build this target (and we have not hit
- * an error running earlier comands) we launch the command using exec_cmd().
- * Command execution signals its completion in exec_wait() by calling our
- * make1c_closure() callback.
- *
- * If there are no more commands to run, we collect the status from all the
- * actions and report our completion to all the parents.
- */
-
-static void make1c( state const * const pState )
-{
- TARGET * const t = pState->t;
- CMD * const cmd = (CMD *)t->cmds;
-
- if ( cmd && t->status == EXEC_CMD_OK )
- {
- /* Pop state first in case something below (e.g. exec_cmd(), exec_wait()
- * or make1c_closure()) pushes a new state. Note that we must not access
- * the popped state data after this as the same stack node might have
- * been reused internally for some newly pushed state.
- */
- pop_state( &state_stack );
-
- /* Increment the jobs running counter. */
- ++cmdsrunning;
-
- /* Execute the actual build command or fake it if no-op. */
- if ( globs.noexec || cmd->noop )
- {
- timing_info time_info = { 0 };
- timestamp_current( &time_info.start );
- timestamp_copy( &time_info.end, &time_info.start );
- make1c_closure( t, EXEC_CMD_OK, &time_info, "", "", EXIT_OK );
- }
- else
- {
- exec_cmd( cmd->buf, make1c_closure, t, cmd->shell );
-
- /* Wait until under the concurrent command count limit. */
- /* FIXME: This wait could be skipped here and moved to just before
- * trying to execute a command that would cross the command count
- * limit. Note though that this might affect the order in which
- * unrelated targets get built and would thus require that all
- * affected Boost Build tests be updated.
- */
- assert( 0 < globs.jobs );
- assert( globs.jobs <= MAXJOBS );
- while ( cmdsrunning >= globs.jobs )
- exec_wait();
- }
- }
- else
- {
- ACTIONS * actions;
-
- /* Collect status from actions, and distribute it as well. */
- for ( actions = t->actions; actions; actions = actions->next )
- if ( actions->action->status > t->status )
- t->status = actions->action->status;
- for ( actions = t->actions; actions; actions = actions->next )
- if ( t->status > actions->action->status )
- actions->action->status = t->status;
-
- /* Tally success/failure for those we tried to update. */
- if ( t->progress == T_MAKE_RUNNING )
- switch ( t->status )
- {
- case EXEC_CMD_OK: ++counts->made; break;
- case EXEC_CMD_FAIL: ++counts->failed; break;
- }
-
- /* Tell parents their dependency has been built. */
- {
- TARGETS * c;
- stack temp_stack = { NULL };
- TARGET * additional_includes = NULL;
-
- t->progress = globs.noexec ? T_MAKE_NOEXEC_DONE : T_MAKE_DONE;
-
- /* Target has been updated so rescan it for dependencies. */
- if ( t->fate >= T_FATE_MISSING && t->status == EXEC_CMD_OK &&
- !( t->flags & T_FLAG_INTERNAL ) )
- {
- TARGET * saved_includes;
- SETTINGS * s;
-
- t->rescanned = 1;
-
- /* Clean current includes. */
- saved_includes = t->includes;
- t->includes = 0;
-
- s = copysettings( t->settings );
- pushsettings( root_module(), s );
- headers( t );
- popsettings( root_module(), s );
- freesettings( s );
-
- if ( t->includes )
- {
- /* Tricky. The parents have already been processed, but they
- * have not seen the internal node, because it was just
- * created. We need to:
- * - push MAKE1A states that would have been pushed by the
- * parents here
- * - make sure all unprocessed parents will pick up the
- * new includes
- * - make sure processing the additional MAKE1A states is
- * done before processing the MAKE1B state for our
- * current target (which would mean this target has
- * already been built), otherwise the parent would be
- * considered built before the additional MAKE1A state
- * processing even got a chance to start.
- */
- make0( t->includes, t->parents->target, 0, 0, 0, t->includes
- );
- /* Link the old includes on to make sure that it gets
- * cleaned up correctly.
- */
- t->includes->includes = saved_includes;
- for ( c = t->dependants; c; c = c->next )
- c->target->depends = targetentry( c->target->depends,
- t->includes );
- /* Will be processed below. */
- additional_includes = t->includes;
- }
- else
- {
- t->includes = saved_includes;
- }
- }
-
- if ( additional_includes )
- for ( c = t->parents; c; c = c->next )
- push_state( &temp_stack, additional_includes, c->target,
- T_STATE_MAKE1A );
-
- if ( t->scc_root )
- {
- TARGET * const scc_root = target_scc( t );
- assert( scc_root->progress < T_MAKE_DONE );
- for ( c = t->parents; c; c = c->next )
- {
- if ( target_scc( c->target ) == scc_root )
- push_state( &temp_stack, c->target, NULL, T_STATE_MAKE1B
- );
- else
- scc_root->parents = targetentry( scc_root->parents,
- c->target );
- }
- }
- else
- {
- for ( c = t->parents; c; c = c->next )
- push_state( &temp_stack, c->target, NULL, T_STATE_MAKE1B );
- }
-
-#ifdef OPT_SEMAPHORE
- /* If there is a semaphore, it is now free. */
- if ( t->semaphore )
- {
- assert( t->semaphore->asynccnt == 1 );
- --t->semaphore->asynccnt;
-
- if ( DEBUG_EXECCMD )
- printf( "SEM: %s is now free\n", object_str(
- t->semaphore->name ) );
-
- /* If anything is waiting, notify the next target. There is no
- * point in notifying all waiting targets, since they will be
- * notified again.
- */
- if ( t->semaphore->parents )
- {
- TARGETS * first = t->semaphore->parents;
- t->semaphore->parents = first->next;
- if ( first->next )
- first->next->tail = first->tail;
-
- if ( DEBUG_EXECCMD )
- printf( "SEM: placing %s on stack\n", object_str(
- first->target->name ) );
- push_state( &temp_stack, first->target, NULL, T_STATE_MAKE1B
- );
- BJAM_FREE( first );
- }
- }
-#endif
-
- /* Must pop state before pushing any more. */
- pop_state( &state_stack );
-
- /* Using stacks reverses the order of execution. Reverse it back. */
- push_stack_on_stack( &state_stack, &temp_stack );
- }
- }
-}
-
-
-/*
- * call_timing_rule() - Look up the __TIMING_RULE__ variable on the given
- * target, and if non-empty, invoke the rule it names, passing the given
- * timing_info.
- */
-
-static void call_timing_rule( TARGET * target, timing_info const * const time )
-{
- LIST * timing_rule;
-
- pushsettings( root_module(), target->settings );
- timing_rule = var_get( root_module(), constant_TIMING_RULE );
- popsettings( root_module(), target->settings );
-
- if ( !list_empty( timing_rule ) )
- {
- /* rule timing-rule ( args * : target : start end user system ) */
-
- /* Prepare the argument list. */
- FRAME frame[ 1 ];
- OBJECT * rulename = list_front( timing_rule );
- frame_init( frame );
-
- /* args * :: $(__TIMING_RULE__[2-]) */
- lol_add( frame->args, list_copy_range( timing_rule, list_next(
- list_begin( timing_rule ) ), list_end( timing_rule ) ) );
-
- /* target :: the name of the target */
- lol_add( frame->args, list_new( object_copy( target->name ) ) );
-
- /* start end user system :: info about the action command */
- lol_add( frame->args, list_push_back( list_push_back( list_push_back( list_new(
- outf_time( &time->start ) ),
- outf_time( &time->end ) ),
- outf_double( time->user ) ),
- outf_double( time->system ) ) );
-
- /* Call the rule. */
- evaluate_rule( bindrule( rulename , root_module() ), rulename, frame );
-
- /* Clean up. */
- frame_free( frame );
- }
-}
-
-
-/*
- * call_action_rule() - Look up the __ACTION_RULE__ variable on the given
- * target, and if non-empty, invoke the rule it names, passing the given info,
- * timing_info, executed command and command output.
- */
-
-static void call_action_rule
-(
- TARGET * target,
- int status,
- timing_info const * time,
- char const * executed_command,
- char const * command_output
-)
-{
- LIST * action_rule;
-
- pushsettings( root_module(), target->settings );
- action_rule = var_get( root_module(), constant_ACTION_RULE );
- popsettings( root_module(), target->settings );
-
- if ( !list_empty( action_rule ) )
- {
- /* rule action-rule (
- args * :
- target :
- command status start end user system :
- output ? ) */
-
- /* Prepare the argument list. */
- FRAME frame[ 1 ];
- OBJECT * rulename = list_front( action_rule );
- frame_init( frame );
-
- /* args * :: $(__ACTION_RULE__[2-]) */
- lol_add( frame->args, list_copy_range( action_rule, list_next(
- list_begin( action_rule ) ), list_end( action_rule ) ) );
-
- /* target :: the name of the target */
- lol_add( frame->args, list_new( object_copy( target->name ) ) );
-
- /* command status start end user system :: info about the action command
- */
- lol_add( frame->args,
- list_push_back( list_push_back( list_push_back( list_push_back( list_push_back( list_new(
- object_new( executed_command ) ),
- outf_int( status ) ),
- outf_time( &time->start ) ),
- outf_time( &time->end ) ),
- outf_double( time->user ) ),
- outf_double( time->system ) ) );
-
- /* output ? :: the output of the action command */
- if ( command_output )
- lol_add( frame->args, list_new( object_new( command_output ) ) );
- else
- lol_add( frame->args, L0 );
-
- /* Call the rule. */
- evaluate_rule( bindrule( rulename, root_module() ), rulename, frame );
-
- /* Clean up. */
- frame_free( frame );
- }
-}
-
-
-/*
- * make1c_closure() - handle command execution completion and go to MAKE1C.
- *
- * Internal function passed as a notification callback for when a command
- * finishes getting executed by the OS or called directly when faking that a
- * command had been executed by the OS.
- *
- * Now all we need to do is fiddle with the command exit status and push a new
- * MAKE1C state to execute the next command scheduled for building this target
- * or close up the target's build process in case there are no more commands
- * scheduled for it. On interrupts, we bail heavily.
- */
-
-static void make1c_closure
-(
- void * const closure,
- int status_orig,
- timing_info const * const time,
- char const * const cmd_stdout,
- char const * const cmd_stderr,
- int const cmd_exit_reason
-)
-{
- TARGET * const t = (TARGET *)closure;
- CMD * const cmd = (CMD *)t->cmds;
- char const * rule_name = 0;
- char const * target_name = 0;
-
- assert( cmd );
-
- --cmdsrunning;
-
- /* Calculate the target's status from the cmd execution result. */
- {
- /* Store the target's status. */
- t->status = status_orig;
-
- /* Invert OK/FAIL target status when FAIL_EXPECTED has been applied. */
- if ( t->flags & T_FLAG_FAIL_EXPECTED && !globs.noexec )
- {
- switch ( t->status )
- {
- case EXEC_CMD_FAIL: t->status = EXEC_CMD_OK; break;
- case EXEC_CMD_OK: t->status = EXEC_CMD_FAIL; break;
- }
- }
-
- /* Ignore failures for actions marked as 'ignore'. */
- if ( t->status == EXEC_CMD_FAIL && cmd->rule->actions->flags &
- RULE_IGNORE )
- t->status = EXEC_CMD_OK;
- }
-
- if ( DEBUG_MAKEQ ||
- ( DEBUG_MAKE && !( cmd->rule->actions->flags & RULE_QUIETLY ) ) )
- {
- rule_name = object_str( cmd->rule->name );
- target_name = object_str( list_front( lol_get( (LOL *)&cmd->args, 0 ) )
- );
- }
-
- out_action( rule_name, target_name, cmd->buf->value, cmd_stdout, cmd_stderr,
- cmd_exit_reason );
-
- if ( !globs.noexec )
- {
- call_timing_rule( t, time );
- if ( DEBUG_EXECCMD )
- printf( "%f sec system; %f sec user\n", time->system, time->user );
-
- /* Assume -p0 is in effect, i.e. cmd_stdout contains merged output. */
- call_action_rule( t, status_orig, time, cmd->buf->value, cmd_stdout );
- }
-
- /* Print command text on failure. */
- if ( t->status == EXEC_CMD_FAIL && DEBUG_MAKE )
- {
- if ( !DEBUG_EXEC )
- printf( "%s\n", cmd->buf->value );
-
- printf( "...failed %s ", object_str( cmd->rule->name ) );
- list_print( lol_get( (LOL *)&cmd->args, 0 ) );
- printf( "...\n" );
- }
-
- /* On interrupt, set quit so _everything_ fails. Do the same for failed
- * commands if we were asked to stop the build in case of any errors.
- */
- if ( t->status == EXEC_CMD_INTR )
- {
- ++intr;
- ++quit;
- }
- if ( t->status == EXEC_CMD_FAIL && globs.quitquick )
- ++quit;
-
- /* If the command was not successful remove all of its targets not marked as
- * "precious".
- */
- if ( t->status != EXEC_CMD_OK )
- {
- LIST * const targets = lol_get( (LOL *)&cmd->args, 0 );
- LISTITER iter = list_begin( targets );
- LISTITER const end = list_end( targets );
- for ( ; iter != end; iter = list_next( iter ) )
- {
- char const * const filename = object_str( list_item( iter ) );
- TARGET const * const t = bindtarget( list_item( iter ) );
- if ( !( t->flags & T_FLAG_PRECIOUS ) && !unlink( filename ) )
- printf( "...removing %s\n", filename );
- }
- }
-
- /* Free this command and push the MAKE1C state to execute the next one
- * scheduled for building this same target.
- */
- t->cmds = (char *)cmd_next( cmd );
- cmd_free( cmd );
- push_state( &state_stack, t, NULL, T_STATE_MAKE1C );
-}
-
-
-/*
- * swap_settings() - replace the settings from the current module and target
- * with those from the new module and target
- */
-
-static void swap_settings
-(
- module_t * * current_module,
- TARGET * * current_target,
- module_t * new_module,
- TARGET * new_target
-)
-{
- if ( ( new_target == *current_target ) &&
- ( new_module == *current_module ) )
- return;
-
- if ( *current_target )
- popsettings( *current_module, (*current_target)->settings );
-
- if ( new_target )
- pushsettings( new_module, new_target->settings );
-
- *current_module = new_module;
- *current_target = new_target;
-}
-
-
-/*
- * make1cmds() - turn ACTIONS into CMDs, grouping, splitting, etc.
- *
- * Essentially copies a chain of ACTIONs to a chain of CMDs, grouping
- * RULE_TOGETHER actions, splitting RULE_PIECEMEAL actions, and handling
- * RULE_NEWSRCS actions. The result is a chain of CMDs which has already had all
- * of its embedded variable references expanded and can now be executed using
- * exec_cmd().
- */
-
-static CMD * make1cmds( TARGET * t )
-{
- CMD * cmds = 0;
- CMD * * cmds_next = &cmds;
- LIST * shell = L0;
- module_t * settings_module = 0;
- TARGET * settings_target = 0;
- ACTIONS * a0;
- int const running_flag = globs.noexec ? A_RUNNING_NOEXEC : A_RUNNING;
-
- /* Step through actions. Actions may be shared with other targets or grouped
- * using RULE_TOGETHER, so actions already seen are skipped.
- */
- for ( a0 = t->actions; a0; a0 = a0->next )
- {
- RULE * rule = a0->action->rule;
- rule_actions * actions = rule->actions;
- SETTINGS * boundvars;
- LIST * nt;
- LIST * ns;
- ACTIONS * a1;
-
- /* Only do rules with commands to execute. If this action has already
- * been executed, use saved status.
- */
- if ( !actions || a0->action->running >= running_flag )
- continue;
-
- a0->action->running = running_flag;
-
- /* Make LISTS of targets and sources. If `execute together` has been
- * specified for this rule, tack on sources from each instance of this
- * rule for this target.
- */
- nt = make1list( L0, a0->action->targets, 0 );
- ns = make1list( L0, a0->action->sources, actions->flags );
- if ( actions->flags & RULE_TOGETHER )
- for ( a1 = a0->next; a1; a1 = a1->next )
- if ( a1->action->rule == rule &&
- a1->action->running < running_flag )
- {
- ns = make1list( ns, a1->action->sources, actions->flags );
- a1->action->running = running_flag;
- }
-
- /* If doing only updated (or existing) sources, but none have been
- * updated (or exist), skip this action.
- */
- if ( list_empty( ns ) &&
- ( actions->flags & ( RULE_NEWSRCS | RULE_EXISTING ) ) )
- {
- list_free( nt );
- continue;
- }
-
- swap_settings( &settings_module, &settings_target, rule->module, t );
- if ( list_empty( shell ) )
- {
- /* shell is per-target */
- shell = var_get( rule->module, constant_JAMSHELL );
- }
-
- /* If we had 'actions xxx bind vars' we bind the vars now. */
- boundvars = make1settings( rule->module, actions->bindlist );
- pushsettings( rule->module, boundvars );
-
- /*
- * Build command, starting with all source args.
- *
- * For actions that allow PIECEMEAL commands, if the constructed command
- * string is too long, we retry constructing it with a reduced number of
- * source arguments presented.
- *
- * While reducing slowly takes a bit of compute time to get things just
- * right, it is worth it to get as close to maximum allowed command
- * string length as possible, because launching the commands we are
- * executing is likely to be much more compute intensive.
- *
- * Note that we loop through at least once, for sourceless actions.
- */
- {
- int const length = list_length( ns );
- int start = 0;
- int chunk = length;
- LIST * cmd_targets = L0;
- LIST * cmd_shell = L0;
- do
- {
- CMD * cmd;
- int cmd_check_result;
- int cmd_error_length;
- int cmd_error_max_length;
- int retry = 0;
- int accept_command = 0;
-
- /* Build cmd: cmd_new() takes ownership of its lists. */
- if ( list_empty( cmd_targets ) ) cmd_targets = list_copy( nt );
- if ( list_empty( cmd_shell ) ) cmd_shell = list_copy( shell );
- cmd = cmd_new( rule, cmd_targets, list_sublist( ns, start,
- chunk ), cmd_shell );
-
- cmd_check_result = exec_check( cmd->buf, &cmd->shell,
- &cmd_error_length, &cmd_error_max_length );
-
- if ( cmd_check_result == EXEC_CHECK_OK )
- {
- accept_command = 1;
- }
- else if ( cmd_check_result == EXEC_CHECK_NOOP )
- {
- accept_command = 1;
- cmd->noop = 1;
- }
- else if ( ( actions->flags & RULE_PIECEMEAL ) && ( chunk > 1 ) )
- {
- /* Too long but splittable. Reduce chunk size slowly and
- * retry.
- */
- assert( cmd_check_result == EXEC_CHECK_TOO_LONG ||
- cmd_check_result == EXEC_CHECK_LINE_TOO_LONG );
- chunk = chunk * 9 / 10;
- retry = 1;
- }
- else
- {
- /* Too long and not splittable. */
- char const * const error_message = cmd_check_result ==
- EXEC_CHECK_TOO_LONG
- ? "is too long"
- : "contains a line that is too long";
- assert( cmd_check_result == EXEC_CHECK_TOO_LONG ||
- cmd_check_result == EXEC_CHECK_LINE_TOO_LONG );
- printf( "%s action %s (%d, max %d):\n", object_str(
- rule->name ), error_message, cmd_error_length,
- cmd_error_max_length );
-
- /* Tell the user what did not fit. */
- fputs( cmd->buf->value, stdout );
- exit( EXITBAD );
- }
-
- assert( !retry || !accept_command );
-
- if ( accept_command )
- {
- /* Chain it up. */
- *cmds_next = cmd;
- cmds_next = &cmd->next;
-
- /* Mark lists we need recreated for the next command since
- * they got consumed by the cmd object.
- */
- cmd_targets = L0;
- cmd_shell = L0;
- }
- else
- {
- /* We can reuse targets & shell lists for the next command
- * if we do not let them die with this cmd object.
- */
- cmd_release_targets_and_shell( cmd );
- cmd_free( cmd );
- }
-
- if ( !retry )
- start += chunk;
- }
- while ( start < length );
- }
-
- /* These were always copied when used. */
- list_free( nt );
- list_free( ns );
-
- /* Free variables with values bound by 'actions xxx bind vars'. */
- popsettings( rule->module, boundvars );
- freesettings( boundvars );
- }
-
- swap_settings( &settings_module, &settings_target, 0, 0 );
- return cmds;
-}
-
-
-/*
- * make1list() - turn a list of targets into a LIST, for $(<) and $(>)
- */
-
-static LIST * make1list( LIST * l, TARGETS * targets, int flags )
-{
- for ( ; targets; targets = targets->next )
- {
- TARGET * t = targets->target;
-
- if ( t->binding == T_BIND_UNBOUND )
- make1bind( t );
-
- if ( ( flags & RULE_EXISTING ) && ( flags & RULE_NEWSRCS ) )
- {
- if ( ( t->binding != T_BIND_EXISTS ) &&
- ( t->fate <= T_FATE_STABLE ) )
- continue;
- }
- else if ( flags & RULE_EXISTING )
- {
- if ( t->binding != T_BIND_EXISTS )
- continue;
- }
- else if ( flags & RULE_NEWSRCS )
- {
- if ( t->fate <= T_FATE_STABLE )
- continue;
- }
-
- /* Prohibit duplicates for RULE_TOGETHER. */
- if ( flags & RULE_TOGETHER )
- {
- LISTITER iter = list_begin( l );
- LISTITER const end = list_end( l );
- for ( ; iter != end; iter = list_next( iter ) )
- if ( object_equal( list_item( iter ), t->boundname ) )
- break;
- if ( iter != end )
- continue;
- }
-
- /* Build new list. */
- l = list_push_back( l, object_copy( t->boundname ) );
- }
-
- return l;
-}
-
-
-/*
- * make1settings() - for vars with bound values, build up replacement lists
- */
-
-static SETTINGS * make1settings( struct module_t * module, LIST * vars )
-{
- SETTINGS * settings = 0;
-
- LISTITER vars_iter = list_begin( vars );
- LISTITER const vars_end = list_end( vars );
- for ( ; vars_iter != vars_end; vars_iter = list_next( vars_iter ) )
- {
- LIST * const l = var_get( module, list_item( vars_iter ) );
- LIST * nl = L0;
- LISTITER iter = list_begin( l );
- LISTITER const end = list_end( l );
-
- for ( ; iter != end; iter = list_next( iter ) )
- {
- TARGET * const t = bindtarget( list_item( iter ) );
-
- /* Make sure the target is bound. */
- if ( t->binding == T_BIND_UNBOUND )
- make1bind( t );
-
- /* Build a new list. */
- nl = list_push_back( nl, object_copy( t->boundname ) );
- }
-
- /* Add to settings chain. */
- settings = addsettings( settings, VAR_SET, list_item( vars_iter ), nl );
- }
-
- return settings;
-}
-
-
-/*
- * make1bind() - bind targets that were not bound during dependency analysis
- *
- * Spot the kludge! If a target is not in the dependency tree, it did not get
- * bound by make0(), so we have to do it here. Ugly.
- */
-
-static void make1bind( TARGET * t )
-{
- if ( t->flags & T_FLAG_NOTFILE )
- return;
-
- pushsettings( root_module(), t->settings );
- object_free( t->boundname );
- t->boundname = search( t->name, &t->time, 0, t->flags & T_FLAG_ISFILE );
- t->binding = timestamp_empty( &t->time ) ? T_BIND_MISSING : T_BIND_EXISTS;
- popsettings( root_module(), t->settings );
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/md5.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/md5.c b/ext/kenlm/jam-files/engine/md5.c
deleted file mode 100644
index c35d96c..0000000
--- a/ext/kenlm/jam-files/engine/md5.c
+++ /dev/null
@@ -1,381 +0,0 @@
-/*
- Copyright (C) 1999, 2000, 2002 Aladdin Enterprises. All rights reserved.
-
- This software is provided 'as-is', without any express or implied
- warranty. In no event will the authors be held liable for any damages
- arising from the use of this software.
-
- Permission is granted to anyone to use this software for any purpose,
- including commercial applications, and to alter it and redistribute it
- freely, subject to the following restrictions:
-
- 1. The origin of this software must not be misrepresented; you must not
- claim that you wrote the original software. If you use this software
- in a product, an acknowledgment in the product documentation would be
- appreciated but is not required.
- 2. Altered source versions must be plainly marked as such, and must not be
- misrepresented as being the original software.
- 3. This notice may not be removed or altered from any source distribution.
-
- L. Peter Deutsch
- ghost@aladdin.com
-
- */
-/* $Id: md5.c,v 1.6 2002/04/13 19:20:28 lpd Exp $ */
-/*
- Independent implementation of MD5 (RFC 1321).
-
- This code implements the MD5 Algorithm defined in RFC 1321, whose
- text is available at
- http://www.ietf.org/rfc/rfc1321.txt
- The code is derived from the text of the RFC, including the test suite
- (section A.5) but excluding the rest of Appendix A. It does not include
- any code or documentation that is identified in the RFC as being
- copyrighted.
-
- The original and principal author of md5.c is L. Peter Deutsch
- <gh...@aladdin.com>. Other authors are noted in the change history
- that follows (in reverse chronological order):
-
- 2002-04-13 lpd Clarified derivation from RFC 1321; now handles byte order
- either statically or dynamically; added missing #include <string.h>
- in library.
- 2002-03-11 lpd Corrected argument list for main(), and added int return
- type, in test program and T value program.
- 2002-02-21 lpd Added missing #include <stdio.h> in test program.
- 2000-07-03 lpd Patched to eliminate warnings about "constant is
- unsigned in ANSI C, signed in traditional"; made test program
- self-checking.
- 1999-11-04 lpd Edited comments slightly for automatic TOC extraction.
- 1999-10-18 lpd Fixed typo in header comment (ansi2knr rather than md5).
- 1999-05-03 lpd Original version.
- */
-
-#include "md5.h"
-#include <string.h>
-
-#undef BYTE_ORDER /* 1 = big-endian, -1 = little-endian, 0 = unknown */
-#ifdef ARCH_IS_BIG_ENDIAN
-# define BYTE_ORDER (ARCH_IS_BIG_ENDIAN ? 1 : -1)
-#else
-# define BYTE_ORDER 0
-#endif
-
-#define T_MASK ((md5_word_t)~0)
-#define T1 /* 0xd76aa478 */ (T_MASK ^ 0x28955b87)
-#define T2 /* 0xe8c7b756 */ (T_MASK ^ 0x173848a9)
-#define T3 0x242070db
-#define T4 /* 0xc1bdceee */ (T_MASK ^ 0x3e423111)
-#define T5 /* 0xf57c0faf */ (T_MASK ^ 0x0a83f050)
-#define T6 0x4787c62a
-#define T7 /* 0xa8304613 */ (T_MASK ^ 0x57cfb9ec)
-#define T8 /* 0xfd469501 */ (T_MASK ^ 0x02b96afe)
-#define T9 0x698098d8
-#define T10 /* 0x8b44f7af */ (T_MASK ^ 0x74bb0850)
-#define T11 /* 0xffff5bb1 */ (T_MASK ^ 0x0000a44e)
-#define T12 /* 0x895cd7be */ (T_MASK ^ 0x76a32841)
-#define T13 0x6b901122
-#define T14 /* 0xfd987193 */ (T_MASK ^ 0x02678e6c)
-#define T15 /* 0xa679438e */ (T_MASK ^ 0x5986bc71)
-#define T16 0x49b40821
-#define T17 /* 0xf61e2562 */ (T_MASK ^ 0x09e1da9d)
-#define T18 /* 0xc040b340 */ (T_MASK ^ 0x3fbf4cbf)
-#define T19 0x265e5a51
-#define T20 /* 0xe9b6c7aa */ (T_MASK ^ 0x16493855)
-#define T21 /* 0xd62f105d */ (T_MASK ^ 0x29d0efa2)
-#define T22 0x02441453
-#define T23 /* 0xd8a1e681 */ (T_MASK ^ 0x275e197e)
-#define T24 /* 0xe7d3fbc8 */ (T_MASK ^ 0x182c0437)
-#define T25 0x21e1cde6
-#define T26 /* 0xc33707d6 */ (T_MASK ^ 0x3cc8f829)
-#define T27 /* 0xf4d50d87 */ (T_MASK ^ 0x0b2af278)
-#define T28 0x455a14ed
-#define T29 /* 0xa9e3e905 */ (T_MASK ^ 0x561c16fa)
-#define T30 /* 0xfcefa3f8 */ (T_MASK ^ 0x03105c07)
-#define T31 0x676f02d9
-#define T32 /* 0x8d2a4c8a */ (T_MASK ^ 0x72d5b375)
-#define T33 /* 0xfffa3942 */ (T_MASK ^ 0x0005c6bd)
-#define T34 /* 0x8771f681 */ (T_MASK ^ 0x788e097e)
-#define T35 0x6d9d6122
-#define T36 /* 0xfde5380c */ (T_MASK ^ 0x021ac7f3)
-#define T37 /* 0xa4beea44 */ (T_MASK ^ 0x5b4115bb)
-#define T38 0x4bdecfa9
-#define T39 /* 0xf6bb4b60 */ (T_MASK ^ 0x0944b49f)
-#define T40 /* 0xbebfbc70 */ (T_MASK ^ 0x4140438f)
-#define T41 0x289b7ec6
-#define T42 /* 0xeaa127fa */ (T_MASK ^ 0x155ed805)
-#define T43 /* 0xd4ef3085 */ (T_MASK ^ 0x2b10cf7a)
-#define T44 0x04881d05
-#define T45 /* 0xd9d4d039 */ (T_MASK ^ 0x262b2fc6)
-#define T46 /* 0xe6db99e5 */ (T_MASK ^ 0x1924661a)
-#define T47 0x1fa27cf8
-#define T48 /* 0xc4ac5665 */ (T_MASK ^ 0x3b53a99a)
-#define T49 /* 0xf4292244 */ (T_MASK ^ 0x0bd6ddbb)
-#define T50 0x432aff97
-#define T51 /* 0xab9423a7 */ (T_MASK ^ 0x546bdc58)
-#define T52 /* 0xfc93a039 */ (T_MASK ^ 0x036c5fc6)
-#define T53 0x655b59c3
-#define T54 /* 0x8f0ccc92 */ (T_MASK ^ 0x70f3336d)
-#define T55 /* 0xffeff47d */ (T_MASK ^ 0x00100b82)
-#define T56 /* 0x85845dd1 */ (T_MASK ^ 0x7a7ba22e)
-#define T57 0x6fa87e4f
-#define T58 /* 0xfe2ce6e0 */ (T_MASK ^ 0x01d3191f)
-#define T59 /* 0xa3014314 */ (T_MASK ^ 0x5cfebceb)
-#define T60 0x4e0811a1
-#define T61 /* 0xf7537e82 */ (T_MASK ^ 0x08ac817d)
-#define T62 /* 0xbd3af235 */ (T_MASK ^ 0x42c50dca)
-#define T63 0x2ad7d2bb
-#define T64 /* 0xeb86d391 */ (T_MASK ^ 0x14792c6e)
-
-
-static void
-md5_process(md5_state_t *pms, const md5_byte_t *data /*[64]*/)
-{
- md5_word_t
- a = pms->abcd[0], b = pms->abcd[1],
- c = pms->abcd[2], d = pms->abcd[3];
- md5_word_t t;
-#if BYTE_ORDER > 0
- /* Define storage only for big-endian CPUs. */
- md5_word_t X[16];
-#else
- /* Define storage for little-endian or both types of CPUs. */
- md5_word_t xbuf[16];
- const md5_word_t *X;
-#endif
-
- {
-#if BYTE_ORDER == 0
- /*
- * Determine dynamically whether this is a big-endian or
- * little-endian machine, since we can use a more efficient
- * algorithm on the latter.
- */
- static const int w = 1;
-
- if (*((const md5_byte_t *)&w)) /* dynamic little-endian */
-#endif
-#if BYTE_ORDER <= 0 /* little-endian */
- {
- /*
- * On little-endian machines, we can process properly aligned
- * data without copying it.
- */
- if (!((data - (const md5_byte_t *)0) & 3)) {
- /* data are properly aligned */
- X = (const md5_word_t *)data;
- } else {
- /* not aligned */
- memcpy(xbuf, data, 64);
- X = xbuf;
- }
- }
-#endif
-#if BYTE_ORDER == 0
- else /* dynamic big-endian */
-#endif
-#if BYTE_ORDER >= 0 /* big-endian */
- {
- /*
- * On big-endian machines, we must arrange the bytes in the
- * right order.
- */
- const md5_byte_t *xp = data;
- int i;
-
-# if BYTE_ORDER == 0
- X = xbuf; /* (dynamic only) */
-# else
-# define xbuf X /* (static only) */
-# endif
- for (i = 0; i < 16; ++i, xp += 4)
- xbuf[i] = xp[0] + (xp[1] << 8) + (xp[2] << 16) + (xp[3] << 24);
- }
-#endif
- }
-
-#define ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32 - (n))))
-
- /* Round 1. */
- /* Let [abcd k s i] denote the operation
- a = b + ((a + F(b,c,d) + X[k] + T[i]) <<< s). */
-#define F(x, y, z) (((x) & (y)) | (~(x) & (z)))
-#define SET(a, b, c, d, k, s, Ti)\
- t = a + F(b,c,d) + X[k] + Ti;\
- a = ROTATE_LEFT(t, s) + b
- /* Do the following 16 operations. */
- SET(a, b, c, d, 0, 7, T1);
- SET(d, a, b, c, 1, 12, T2);
- SET(c, d, a, b, 2, 17, T3);
- SET(b, c, d, a, 3, 22, T4);
- SET(a, b, c, d, 4, 7, T5);
- SET(d, a, b, c, 5, 12, T6);
- SET(c, d, a, b, 6, 17, T7);
- SET(b, c, d, a, 7, 22, T8);
- SET(a, b, c, d, 8, 7, T9);
- SET(d, a, b, c, 9, 12, T10);
- SET(c, d, a, b, 10, 17, T11);
- SET(b, c, d, a, 11, 22, T12);
- SET(a, b, c, d, 12, 7, T13);
- SET(d, a, b, c, 13, 12, T14);
- SET(c, d, a, b, 14, 17, T15);
- SET(b, c, d, a, 15, 22, T16);
-#undef SET
-
- /* Round 2. */
- /* Let [abcd k s i] denote the operation
- a = b + ((a + G(b,c,d) + X[k] + T[i]) <<< s). */
-#define G(x, y, z) (((x) & (z)) | ((y) & ~(z)))
-#define SET(a, b, c, d, k, s, Ti)\
- t = a + G(b,c,d) + X[k] + Ti;\
- a = ROTATE_LEFT(t, s) + b
- /* Do the following 16 operations. */
- SET(a, b, c, d, 1, 5, T17);
- SET(d, a, b, c, 6, 9, T18);
- SET(c, d, a, b, 11, 14, T19);
- SET(b, c, d, a, 0, 20, T20);
- SET(a, b, c, d, 5, 5, T21);
- SET(d, a, b, c, 10, 9, T22);
- SET(c, d, a, b, 15, 14, T23);
- SET(b, c, d, a, 4, 20, T24);
- SET(a, b, c, d, 9, 5, T25);
- SET(d, a, b, c, 14, 9, T26);
- SET(c, d, a, b, 3, 14, T27);
- SET(b, c, d, a, 8, 20, T28);
- SET(a, b, c, d, 13, 5, T29);
- SET(d, a, b, c, 2, 9, T30);
- SET(c, d, a, b, 7, 14, T31);
- SET(b, c, d, a, 12, 20, T32);
-#undef SET
-
- /* Round 3. */
- /* Let [abcd k s t] denote the operation
- a = b + ((a + H(b,c,d) + X[k] + T[i]) <<< s). */
-#define H(x, y, z) ((x) ^ (y) ^ (z))
-#define SET(a, b, c, d, k, s, Ti)\
- t = a + H(b,c,d) + X[k] + Ti;\
- a = ROTATE_LEFT(t, s) + b
- /* Do the following 16 operations. */
- SET(a, b, c, d, 5, 4, T33);
- SET(d, a, b, c, 8, 11, T34);
- SET(c, d, a, b, 11, 16, T35);
- SET(b, c, d, a, 14, 23, T36);
- SET(a, b, c, d, 1, 4, T37);
- SET(d, a, b, c, 4, 11, T38);
- SET(c, d, a, b, 7, 16, T39);
- SET(b, c, d, a, 10, 23, T40);
- SET(a, b, c, d, 13, 4, T41);
- SET(d, a, b, c, 0, 11, T42);
- SET(c, d, a, b, 3, 16, T43);
- SET(b, c, d, a, 6, 23, T44);
- SET(a, b, c, d, 9, 4, T45);
- SET(d, a, b, c, 12, 11, T46);
- SET(c, d, a, b, 15, 16, T47);
- SET(b, c, d, a, 2, 23, T48);
-#undef SET
-
- /* Round 4. */
- /* Let [abcd k s t] denote the operation
- a = b + ((a + I(b,c,d) + X[k] + T[i]) <<< s). */
-#define I(x, y, z) ((y) ^ ((x) | ~(z)))
-#define SET(a, b, c, d, k, s, Ti)\
- t = a + I(b,c,d) + X[k] + Ti;\
- a = ROTATE_LEFT(t, s) + b
- /* Do the following 16 operations. */
- SET(a, b, c, d, 0, 6, T49);
- SET(d, a, b, c, 7, 10, T50);
- SET(c, d, a, b, 14, 15, T51);
- SET(b, c, d, a, 5, 21, T52);
- SET(a, b, c, d, 12, 6, T53);
- SET(d, a, b, c, 3, 10, T54);
- SET(c, d, a, b, 10, 15, T55);
- SET(b, c, d, a, 1, 21, T56);
- SET(a, b, c, d, 8, 6, T57);
- SET(d, a, b, c, 15, 10, T58);
- SET(c, d, a, b, 6, 15, T59);
- SET(b, c, d, a, 13, 21, T60);
- SET(a, b, c, d, 4, 6, T61);
- SET(d, a, b, c, 11, 10, T62);
- SET(c, d, a, b, 2, 15, T63);
- SET(b, c, d, a, 9, 21, T64);
-#undef SET
-
- /* Then perform the following additions. (That is increment each
- of the four registers by the value it had before this block
- was started.) */
- pms->abcd[0] += a;
- pms->abcd[1] += b;
- pms->abcd[2] += c;
- pms->abcd[3] += d;
-}
-
-void
-md5_init(md5_state_t *pms)
-{
- pms->count[0] = pms->count[1] = 0;
- pms->abcd[0] = 0x67452301;
- pms->abcd[1] = /*0xefcdab89*/ T_MASK ^ 0x10325476;
- pms->abcd[2] = /*0x98badcfe*/ T_MASK ^ 0x67452301;
- pms->abcd[3] = 0x10325476;
-}
-
-void
-md5_append(md5_state_t *pms, const md5_byte_t *data, int nbytes)
-{
- const md5_byte_t *p = data;
- int left = nbytes;
- int offset = (pms->count[0] >> 3) & 63;
- md5_word_t nbits = (md5_word_t)(nbytes << 3);
-
- if (nbytes <= 0)
- return;
-
- /* Update the message length. */
- pms->count[1] += nbytes >> 29;
- pms->count[0] += nbits;
- if (pms->count[0] < nbits)
- pms->count[1]++;
-
- /* Process an initial partial block. */
- if (offset) {
- int copy = (offset + nbytes > 64 ? 64 - offset : nbytes);
-
- memcpy(pms->buf + offset, p, copy);
- if (offset + copy < 64)
- return;
- p += copy;
- left -= copy;
- md5_process(pms, pms->buf);
- }
-
- /* Process full blocks. */
- for (; left >= 64; p += 64, left -= 64)
- md5_process(pms, p);
-
- /* Process a final partial block. */
- if (left)
- memcpy(pms->buf, p, left);
-}
-
-void
-md5_finish(md5_state_t *pms, md5_byte_t digest[16])
-{
- static const md5_byte_t pad[64] = {
- 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
- };
- md5_byte_t data[8];
- int i;
-
- /* Save the length before padding. */
- for (i = 0; i < 8; ++i)
- data[i] = (md5_byte_t)(pms->count[i >> 2] >> ((i & 3) << 3));
- /* Pad to 56 bytes mod 64. */
- md5_append(pms, pad, ((55 - (pms->count[0] >> 3)) & 63) + 1);
- /* Append the length. */
- md5_append(pms, data, 8);
- for (i = 0; i < 16; ++i)
- digest[i] = (md5_byte_t)(pms->abcd[i >> 2] >> ((i & 3) << 3));
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/md5.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/md5.h b/ext/kenlm/jam-files/engine/md5.h
deleted file mode 100644
index 698c995..0000000
--- a/ext/kenlm/jam-files/engine/md5.h
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- Copyright (C) 1999, 2002 Aladdin Enterprises. All rights reserved.
-
- This software is provided 'as-is', without any express or implied
- warranty. In no event will the authors be held liable for any damages
- arising from the use of this software.
-
- Permission is granted to anyone to use this software for any purpose,
- including commercial applications, and to alter it and redistribute it
- freely, subject to the following restrictions:
-
- 1. The origin of this software must not be misrepresented; you must not
- claim that you wrote the original software. If you use this software
- in a product, an acknowledgment in the product documentation would be
- appreciated but is not required.
- 2. Altered source versions must be plainly marked as such, and must not be
- misrepresented as being the original software.
- 3. This notice may not be removed or altered from any source distribution.
-
- L. Peter Deutsch
- ghost@aladdin.com
-
- */
-/* $Id: md5.h,v 1.4 2002/04/13 19:20:28 lpd Exp $ */
-/*
- Independent implementation of MD5 (RFC 1321).
-
- This code implements the MD5 Algorithm defined in RFC 1321, whose
- text is available at
- http://www.ietf.org/rfc/rfc1321.txt
- The code is derived from the text of the RFC, including the test suite
- (section A.5) but excluding the rest of Appendix A. It does not include
- any code or documentation that is identified in the RFC as being
- copyrighted.
-
- The original and principal author of md5.h is L. Peter Deutsch
- <gh...@aladdin.com>. Other authors are noted in the change history
- that follows (in reverse chronological order):
-
- 2002-04-13 lpd Removed support for non-ANSI compilers; removed
- references to Ghostscript; clarified derivation from RFC 1321;
- now handles byte order either statically or dynamically.
- 1999-11-04 lpd Edited comments slightly for automatic TOC extraction.
- 1999-10-18 lpd Fixed typo in header comment (ansi2knr rather than md5);
- added conditionalization for C++ compilation from Martin
- Purschke <pu...@bnl.gov>.
- 1999-05-03 lpd Original version.
- */
-
-#ifndef md5_INCLUDED
-# define md5_INCLUDED
-
-/*
- * This package supports both compile-time and run-time determination of CPU
- * byte order. If ARCH_IS_BIG_ENDIAN is defined as 0, the code will be
- * compiled to run only on little-endian CPUs; if ARCH_IS_BIG_ENDIAN is
- * defined as non-zero, the code will be compiled to run only on big-endian
- * CPUs; if ARCH_IS_BIG_ENDIAN is not defined, the code will be compiled to
- * run on either big- or little-endian CPUs, but will run slightly less
- * efficiently on either one than if ARCH_IS_BIG_ENDIAN is defined.
- */
-
-typedef unsigned char md5_byte_t; /* 8-bit byte */
-typedef unsigned int md5_word_t; /* 32-bit word */
-
-/* Define the state of the MD5 Algorithm. */
-typedef struct md5_state_s {
- md5_word_t count[2]; /* message length in bits, lsw first */
- md5_word_t abcd[4]; /* digest buffer */
- md5_byte_t buf[64]; /* accumulate block */
-} md5_state_t;
-
-#ifdef __cplusplus
-extern "C"
-{
-#endif
-
-/* Initialize the algorithm. */
-void md5_init(md5_state_t *pms);
-
-/* Append a string to the message. */
-void md5_append(md5_state_t *pms, const md5_byte_t *data, int nbytes);
-
-/* Finish the message and return the digest. */
-void md5_finish(md5_state_t *pms, md5_byte_t digest[16]);
-
-#ifdef __cplusplus
-} /* end extern "C" */
-#endif
-
-#endif /* md5_INCLUDED */
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/mem.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/mem.c b/ext/kenlm/jam-files/engine/mem.c
deleted file mode 100644
index 6a11fb3..0000000
--- a/ext/kenlm/jam-files/engine/mem.c
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
-Copyright Rene Rivera 2006.
-Distributed under the Boost Software License, Version 1.0.
-(See accompanying file LICENSE_1_0.txt or copy at
-http://www.boost.org/LICENSE_1_0.txt)
-*/
-
-#include "jam.h"
-
-#ifdef OPT_BOEHM_GC
-
- /* Compile the Boehm GC as one big chunk of code. It's much easier
- this way, than trying to make radical changes to the bjam build
- scripts. */
-
- #define ATOMIC_UNCOLLECTABLE
- #define NO_EXECUTE_PERMISSION
- #define ALL_INTERIOR_POINTERS
-
- #define LARGE_CONFIG
- /*
- #define NO_SIGNALS
- #define SILENT
- */
- #ifndef GC_DEBUG
- #define NO_DEBUGGING
- #endif
-
- #ifdef __GLIBC__
- #define __USE_GNU
- #endif
-
- #include "boehm_gc/reclaim.c"
- #include "boehm_gc/allchblk.c"
- #include "boehm_gc/misc.c"
- #include "boehm_gc/alloc.c"
- #include "boehm_gc/mach_dep.c"
- #include "boehm_gc/os_dep.c"
- #include "boehm_gc/mark_rts.c"
- #include "boehm_gc/headers.c"
- #include "boehm_gc/mark.c"
- #include "boehm_gc/obj_map.c"
- #include "boehm_gc/pcr_interface.c"
- #include "boehm_gc/blacklst.c"
- #include "boehm_gc/new_hblk.c"
- #include "boehm_gc/real_malloc.c"
- #include "boehm_gc/dyn_load.c"
- #include "boehm_gc/dbg_mlc.c"
- #include "boehm_gc/malloc.c"
- #include "boehm_gc/stubborn.c"
- #include "boehm_gc/checksums.c"
- #include "boehm_gc/pthread_support.c"
- #include "boehm_gc/pthread_stop_world.c"
- #include "boehm_gc/darwin_stop_world.c"
- #include "boehm_gc/typd_mlc.c"
- #include "boehm_gc/ptr_chck.c"
- #include "boehm_gc/mallocx.c"
- #include "boehm_gc/gcj_mlc.c"
- #include "boehm_gc/specific.c"
- #include "boehm_gc/gc_dlopen.c"
- #include "boehm_gc/backgraph.c"
- #include "boehm_gc/win32_threads.c"
-
- /* Needs to be last. */
- #include "boehm_gc/finalize.c"
-
-#elif defined(OPT_DUMA)
-
- #ifdef OS_NT
- #define WIN32
- #endif
- #include "duma/duma.c"
- #include "duma/print.c"
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/mem.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/mem.h b/ext/kenlm/jam-files/engine/mem.h
deleted file mode 100644
index 8718b07..0000000
--- a/ext/kenlm/jam-files/engine/mem.h
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Copyright 2006. Rene Rivera
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#ifndef BJAM_MEM_H
-#define BJAM_MEM_H
-
-#ifdef OPT_BOEHM_GC
-
- /* Use Boehm GC memory allocator. */
- #include <gc.h>
-
- #define bjam_malloc_x(s) memset(GC_malloc(s),0,s)
- #define bjam_malloc_atomic_x(s) memset(GC_malloc_atomic(s),0,s)
- #define bjam_calloc_x(n,s) memset(GC_malloc((n)*(s)),0,(n)*(s))
- #define bjam_calloc_atomic_x(n,s) memset(GC_malloc_atomic((n)*(s)),0,(n)*(s))
- #define bjam_realloc_x(p,s) GC_realloc(p,s)
- #define bjam_free_x(p) GC_free(p)
- #define bjam_mem_init_x() GC_init(); GC_enable_incremental()
-
- #define bjam_malloc_raw_x(s) malloc(s)
- #define bjam_calloc_raw_x(n,s) calloc(n,s)
- #define bjam_realloc_raw_x(p,s) realloc(p,s)
- #define bjam_free_raw_x(p) free(p)
-
- #ifndef BJAM_NEWSTR_NO_ALLOCATE
- # define BJAM_NEWSTR_NO_ALLOCATE
- #endif
-
-#elif defined( OPT_DUMA )
-
- /* Use Duma memory debugging library. */
- #include <stdlib.h>
-
- #define _DUMA_CONFIG_H_
- #define DUMA_NO_GLOBAL_MALLOC_FREE
- #define DUMA_EXPLICIT_INIT
- #define DUMA_NO_THREAD_SAFETY
- #define DUMA_NO_CPP_SUPPORT
- /* #define DUMA_NO_LEAKDETECTION */
- /* #define DUMA_USE_FRAMENO */
- /* #define DUMA_PREFER_ATEXIT */
- /* #define DUMA_OLD_DEL_MACRO */
- /* #define DUMA_NO_HANG_MSG */
- #define DUMA_PAGE_SIZE 4096
- #define DUMA_MIN_ALIGNMENT 1
- /* #define DUMA_GNU_INIT_ATTR 0 */
- typedef unsigned int DUMA_ADDR;
- typedef unsigned int DUMA_SIZE;
- #include <duma.h>
-
- #define bjam_malloc_x(s) malloc(s)
- #define bjam_calloc_x(n,s) calloc(n,s)
- #define bjam_realloc_x(p,s) realloc(p,s)
- #define bjam_free_x(p) free(p)
-
- #ifndef BJAM_NEWSTR_NO_ALLOCATE
- # define BJAM_NEWSTR_NO_ALLOCATE
- #endif
-
-#else
-
- /* Standard C memory allocation. */
- #include <stdlib.h>
-
- #define bjam_malloc_x(s) malloc(s)
- #define bjam_calloc_x(n,s) calloc(n,s)
- #define bjam_realloc_x(p,s) realloc(p,s)
- #define bjam_free_x(p) free(p)
-
-#endif
-
-#ifndef bjam_malloc_atomic_x
- #define bjam_malloc_atomic_x(s) bjam_malloc_x(s)
-#endif
-#ifndef bjam_calloc_atomic_x
- #define bjam_calloc_atomic_x(n,s) bjam_calloc_x(n,s)
-#endif
-#ifndef bjam_mem_init_x
- #define bjam_mem_init_x()
-#endif
-#ifndef bjam_mem_close_x
- #define bjam_mem_close_x()
-#endif
-#ifndef bjam_malloc_raw_x
- #define bjam_malloc_raw_x(s) bjam_malloc_x(s)
-#endif
-#ifndef bjam_calloc_raw_x
- #define bjam_calloc_raw_x(n,s) bjam_calloc_x(n,s)
-#endif
-#ifndef bjam_realloc_raw_x
- #define bjam_realloc_raw_x(p,s) bjam_realloc_x(p,s)
-#endif
-#ifndef bjam_free_raw_x
- #define bjam_free_raw_x(p) bjam_free_x(p)
-#endif
-
-#ifdef OPT_DEBUG_PROFILE
- /* Profile tracing of memory allocations. */
- #include "debug.h"
-
- #define BJAM_MALLOC(s) (profile_memory(s), bjam_malloc_x(s))
- #define BJAM_MALLOC_ATOMIC(s) (profile_memory(s), bjam_malloc_atomic_x(s))
- #define BJAM_CALLOC(n,s) (profile_memory(n*s), bjam_calloc_x(n,s))
- #define BJAM_CALLOC_ATOMIC(n,s) (profile_memory(n*s), bjam_calloc_atomic_x(n,s))
- #define BJAM_REALLOC(p,s) (profile_memory(s), bjam_realloc_x(p,s))
-
- #define BJAM_MALLOC_RAW(s) (profile_memory(s), bjam_malloc_raw_x(s))
- #define BJAM_CALLOC_RAW(n,s) (profile_memory(n*s), bjam_calloc_raw_x(n,s))
- #define BJAM_REALLOC_RAW(p,s) (profile_memory(s), bjam_realloc_raw_x(p,s))
-#else
- /* No mem tracing. */
- #define BJAM_MALLOC(s) bjam_malloc_x(s)
- #define BJAM_MALLOC_ATOMIC(s) bjam_malloc_atomic_x(s)
- #define BJAM_CALLOC(n,s) bjam_calloc_x(n,s)
- #define BJAM_CALLOC_ATOMIC(n,s) bjam_calloc_atomic_x(n,s)
- #define BJAM_REALLOC(p,s) bjam_realloc_x(p,s)
-
- #define BJAM_MALLOC_RAW(s) bjam_malloc_raw_x(s)
- #define BJAM_CALLOC_RAW(n,s) bjam_calloc_raw_x(n,s)
- #define BJAM_REALLOC_RAW(p,s) bjam_realloc_raw_x(p,s)
-#endif
-
-#define BJAM_MEM_INIT() bjam_mem_init_x()
-#define BJAM_MEM_CLOSE() bjam_mem_close_x()
-
-#define BJAM_FREE(p) bjam_free_x(p)
-#define BJAM_FREE_RAW(p) bjam_free_raw_x(p)
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/mkjambase.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/mkjambase.c b/ext/kenlm/jam-files/engine/mkjambase.c
deleted file mode 100644
index cdf5998..0000000
--- a/ext/kenlm/jam-files/engine/mkjambase.c
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Copyright 1993, 1995 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * mkjambase.c - turn Jambase into a big C structure
- *
- * Usage: mkjambase jambase.c Jambase ...
- *
- * Results look like this:
- *
- * char *jambase[] = {
- * "...\n",
- * ...
- * 0 };
- *
- * Handles \'s and "'s specially; knows to delete blank and comment lines.
- *
- */
-
-#include <stdio.h>
-#include <string.h>
-
-
-int main( int argc, char * * argv, char * * envp )
-{
- char buf[ 1024 ];
- FILE * fin;
- FILE * fout;
- char * p;
- int doDotC = 0;
-
- if ( argc < 3 )
- {
- fprintf( stderr, "usage: %s jambase.c Jambase ...\n", argv[ 0 ] );
- return -1;
- }
-
- if ( !( fout = fopen( argv[1], "w" ) ) )
- {
- perror( argv[ 1 ] );
- return -1;
- }
-
- /* If the file ends in .c generate a C source file. */
- if ( ( p = strrchr( argv[1], '.' ) ) && !strcmp( p, ".c" ) )
- doDotC++;
-
- /* Now process the files. */
-
- argc -= 2;
- argv += 2;
-
- if ( doDotC )
- {
- fprintf( fout, "/* Generated by mkjambase from Jambase */\n" );
- fprintf( fout, "char *jambase[] = {\n" );
- }
-
- for ( ; argc--; ++argv )
- {
- if ( !( fin = fopen( *argv, "r" ) ) )
- {
- perror( *argv );
- return -1;
- }
-
- if ( doDotC )
- fprintf( fout, "/* %s */\n", *argv );
- else
- fprintf( fout, "### %s ###\n", *argv );
-
- while ( fgets( buf, sizeof( buf ), fin ) )
- {
- if ( doDotC )
- {
- char * p = buf;
-
- /* Strip leading whitespace. */
- while ( ( *p == ' ' ) || ( *p == '\t' ) || ( *p == '\n' ) )
- ++p;
-
- /* Drop comments and empty lines. */
- if ( ( *p == '#' ) || !*p )
- continue;
-
- /* Copy. */
- putc( '"', fout );
- for ( ; *p && ( *p != '\n' ); ++p )
- switch ( *p )
- {
- case '\\': putc( '\\', fout ); putc( '\\', fout ); break;
- case '"' : putc( '\\', fout ); putc( '"' , fout ); break;
- case '\r': break;
- default: putc( *p, fout ); break;
- }
-
- fprintf( fout, "\\n\",\n" );
- }
- else
- {
- fprintf( fout, "%s", buf );
- }
- }
-
- fclose( fin );
- }
-
- if ( doDotC )
- fprintf( fout, "0 };\n" );
-
- fclose( fout );
-
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/modules.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/modules.c b/ext/kenlm/jam-files/engine/modules.c
deleted file mode 100644
index 6be82fe..0000000
--- a/ext/kenlm/jam-files/engine/modules.c
+++ /dev/null
@@ -1,431 +0,0 @@
-/*
- * Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#include "jam.h"
-#include "modules.h"
-
-#include "hash.h"
-#include "lists.h"
-#include "native.h"
-#include "object.h"
-#include "parse.h"
-#include "rules.h"
-#include "strings.h"
-#include "variable.h"
-
-#include <assert.h>
-#include <string.h>
-
-static struct hash * module_hash = 0;
-static module_t root;
-
-
-module_t * bindmodule( OBJECT * name )
-{
- if ( !name )
- return &root;
-
- {
- PROFILE_ENTER( BINDMODULE );
-
- module_t * m;
- int found;
-
- if ( !module_hash )
- module_hash = hashinit( sizeof( module_t ), "modules" );
-
- m = (module_t *)hash_insert( module_hash, name, &found );
- if ( !found )
- {
- m->name = object_copy( name );
- m->variables = 0;
- m->variable_indices = 0;
- m->num_fixed_variables = 0;
- m->fixed_variables = 0;
- m->rules = 0;
- m->imported_modules = 0;
- m->class_module = 0;
- m->native_rules = 0;
- m->user_module = 0;
- }
-
- PROFILE_EXIT( BINDMODULE );
-
- return m;
- }
-}
-
-
-/*
- * demand_rules() - Get the module's "rules" hash on demand.
- */
-struct hash * demand_rules( module_t * m )
-{
- if ( !m->rules )
- m->rules = hashinit( sizeof( RULE ), "rules" );
- return m->rules;
-}
-
-
-/*
- * delete_module() - wipe out the module's rules and variables.
- */
-
-static void delete_rule_( void * xrule, void * data )
-{
- rule_free( (RULE *)xrule );
-}
-
-
-static void delete_native_rule( void * xrule, void * data )
-{
- native_rule_t * rule = (native_rule_t *)xrule;
- object_free( rule->name );
- if ( rule->procedure )
- function_free( rule->procedure );
-}
-
-
-static void delete_imported_modules( void * xmodule_name, void * data )
-{
- object_free( *(OBJECT * *)xmodule_name );
-}
-
-
-static void free_fixed_variable( void * xvar, void * data );
-
-void delete_module( module_t * m )
-{
- /* Clear out all the rules. */
- if ( m->rules )
- {
- hashenumerate( m->rules, delete_rule_, (void *)0 );
- hash_free( m->rules );
- m->rules = 0;
- }
-
- if ( m->native_rules )
- {
- hashenumerate( m->native_rules, delete_native_rule, (void *)0 );
- hash_free( m->native_rules );
- m->native_rules = 0;
- }
-
- if ( m->variables )
- {
- var_done( m );
- m->variables = 0;
- }
-
- if ( m->fixed_variables )
- {
- int i;
- for ( i = 0; i < m->num_fixed_variables; ++i )
- {
- list_free( m->fixed_variables[ i ] );
- }
- BJAM_FREE( m->fixed_variables );
- m->fixed_variables = 0;
- }
-
- if ( m->variable_indices )
- {
- hashenumerate( m->variable_indices, &free_fixed_variable, (void *)0 );
- hash_free( m->variable_indices );
- m->variable_indices = 0;
- }
-
- if ( m->imported_modules )
- {
- hashenumerate( m->imported_modules, delete_imported_modules, (void *)0 );
- hash_free( m->imported_modules );
- m->imported_modules = 0;
- }
-}
-
-
-struct module_stats
-{
- OBJECT * module_name;
- struct hashstats rules_stats[ 1 ];
- struct hashstats variables_stats[ 1 ];
- struct hashstats variable_indices_stats[ 1 ];
- struct hashstats imported_modules_stats[ 1 ];
-};
-
-
-static void module_stat( struct hash * hp, OBJECT * module, const char * name )
-{
- if ( hp )
- {
- struct hashstats stats[ 1 ];
- string id[ 1 ];
- hashstats_init( stats );
- string_new( id );
- string_append( id, object_str( module ) );
- string_push_back( id, ' ' );
- string_append( id, name );
-
- hashstats_add( stats, hp );
- hashstats_print( stats, id->value );
-
- string_free( id );
- }
-}
-
-
-static void class_module_stat( struct hashstats * stats, OBJECT * module, const char * name )
-{
- if ( stats->item_size )
- {
- string id[ 1 ];
- string_new( id );
- string_append( id, object_str( module ) );
- string_append( id, " object " );
- string_append( id, name );
-
- hashstats_print( stats, id->value );
-
- string_free( id );
- }
-}
-
-
-static void stat_module( void * xmodule, void * data )
-{
- module_t *m = (module_t *)xmodule;
-
- if ( DEBUG_MEM || DEBUG_PROFILE )
- {
- struct hash * class_info = (struct hash *)data;
- if ( m->class_module )
- {
- int found;
- struct module_stats * ms = (struct module_stats *)hash_insert( class_info, m->class_module->name, &found );
- if ( !found )
- {
- ms->module_name = m->class_module->name;
- hashstats_init( ms->rules_stats );
- hashstats_init( ms->variables_stats );
- hashstats_init( ms->variable_indices_stats );
- hashstats_init( ms->imported_modules_stats );
- }
-
- hashstats_add( ms->rules_stats, m->rules );
- hashstats_add( ms->variables_stats, m->variables );
- hashstats_add( ms->variable_indices_stats, m->variable_indices );
- hashstats_add( ms->imported_modules_stats, m->imported_modules );
- }
- else
- {
- module_stat( m->rules, m->name, "rules" );
- module_stat( m->variables, m->name, "variables" );
- module_stat( m->variable_indices, m->name, "fixed variables" );
- module_stat( m->imported_modules, m->name, "imported modules" );
- }
- }
-
- delete_module( m );
- object_free( m->name );
-}
-
-static void print_class_stats( void * xstats, void * data )
-{
- struct module_stats * stats = (struct module_stats *)xstats;
- class_module_stat( stats->rules_stats, stats->module_name, "rules" );
- class_module_stat( stats->variables_stats, stats->module_name, "variables" );
- class_module_stat( stats->variable_indices_stats, stats->module_name, "fixed variables" );
- class_module_stat( stats->imported_modules_stats, stats->module_name, "imported modules" );
-}
-
-
-static void delete_module_( void * xmodule, void * data )
-{
- module_t *m = (module_t *)xmodule;
-
- delete_module( m );
- object_free( m->name );
-}
-
-
-void modules_done()
-{
- if ( DEBUG_MEM || DEBUG_PROFILE )
- {
- struct hash * class_hash = hashinit( sizeof( struct module_stats ), "object info" );
- hashenumerate( module_hash, stat_module, (void *)class_hash );
- hashenumerate( class_hash, print_class_stats, (void *)0 );
- hash_free( class_hash );
- }
- hashenumerate( module_hash, delete_module_, (void *)0 );
- hashdone( module_hash );
- module_hash = 0;
- delete_module( &root );
-}
-
-module_t * root_module()
-{
- return &root;
-}
-
-
-void import_module( LIST * module_names, module_t * target_module )
-{
- PROFILE_ENTER( IMPORT_MODULE );
-
- struct hash * h;
- LISTITER iter;
- LISTITER end;
-
- if ( !target_module->imported_modules )
- target_module->imported_modules = hashinit( sizeof( char * ), "imported"
- );
- h = target_module->imported_modules;
-
- iter = list_begin( module_names );
- end = list_end( module_names );
- for ( ; iter != end; iter = list_next( iter ) )
- {
- int found;
- OBJECT * const s = list_item( iter );
- OBJECT * * const ss = (OBJECT * *)hash_insert( h, s, &found );
- if ( !found )
- *ss = object_copy( s );
- }
-
- PROFILE_EXIT( IMPORT_MODULE );
-}
-
-
-static void add_module_name( void * r_, void * result_ )
-{
- OBJECT * * const r = (OBJECT * *)r_;
- LIST * * const result = (LIST * *)result_;
- *result = list_push_back( *result, object_copy( *r ) );
-}
-
-
-LIST * imported_modules( module_t * module )
-{
- LIST * result = L0;
- if ( module->imported_modules )
- hashenumerate( module->imported_modules, add_module_name, &result );
- return result;
-}
-
-
-FUNCTION * function_bind_variables( FUNCTION *, module_t *, int * counter );
-FUNCTION * function_unbind_variables( FUNCTION * );
-
-struct fixed_variable
-{
- OBJECT * key;
- int n;
-};
-
-struct bind_vars_t
-{
- module_t * module;
- int counter;
-};
-
-
-static void free_fixed_variable( void * xvar, void * data )
-{
- object_free( ( (struct fixed_variable *)xvar )->key );
-}
-
-
-static void bind_variables_for_rule( void * xrule, void * xdata )
-{
- RULE * rule = (RULE *)xrule;
- struct bind_vars_t * data = (struct bind_vars_t *)xdata;
- if ( rule->procedure && rule->module == data->module )
- rule->procedure = function_bind_variables( rule->procedure,
- data->module, &data->counter );
-}
-
-
-void module_bind_variables( struct module_t * m )
-{
- if ( m != root_module() && m->rules )
- {
- struct bind_vars_t data;
- data.module = m;
- data.counter = m->num_fixed_variables;
- hashenumerate( m->rules, &bind_variables_for_rule, &data );
- module_set_fixed_variables( m, data.counter );
- }
-}
-
-
-int module_add_fixed_var( struct module_t * m, OBJECT * name, int * counter )
-{
- struct fixed_variable * v;
- int found;
-
- assert( !m->class_module );
-
- if ( !m->variable_indices )
- m->variable_indices = hashinit( sizeof( struct fixed_variable ), "variable index table" );
-
- v = (struct fixed_variable *)hash_insert( m->variable_indices, name, &found );
- if ( !found )
- {
- v->key = object_copy( name );
- v->n = (*counter)++;
- }
-
- return v->n;
-}
-
-
-LIST * var_get_and_clear_raw( module_t * m, OBJECT * name );
-
-static void load_fixed_variable( void * xvar, void * data )
-{
- struct fixed_variable * var = (struct fixed_variable *)xvar;
- struct module_t * m = (struct module_t *)data;
- if ( var->n >= m->num_fixed_variables )
- m->fixed_variables[ var->n ] = var_get_and_clear_raw( m, var->key );
-}
-
-
-void module_set_fixed_variables( struct module_t * m, int n_variables )
-{
- /* Reallocate */
- struct hash * variable_indices;
- LIST * * fixed_variables = BJAM_MALLOC( n_variables * sizeof( LIST * ) );
- if ( m->fixed_variables )
- {
- memcpy( fixed_variables, m->fixed_variables, m->num_fixed_variables * sizeof( LIST * ) );
- BJAM_FREE( m->fixed_variables );
- }
- m->fixed_variables = fixed_variables;
- variable_indices = m->class_module
- ? m->class_module->variable_indices
- : m->variable_indices;
- if ( variable_indices )
- hashenumerate( variable_indices, &load_fixed_variable, m );
- m->num_fixed_variables = n_variables;
-}
-
-
-int module_get_fixed_var( struct module_t * m_, OBJECT * name )
-{
- struct fixed_variable * v;
- struct module_t * m = m_;
-
- if ( m->class_module )
- m = m->class_module;
-
- if ( !m->variable_indices )
- return -1;
-
- v = (struct fixed_variable *)hash_find( m->variable_indices, name );
- return v && v->n < m_->num_fixed_variables ? v->n : -1;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/modules.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/modules.h b/ext/kenlm/jam-files/engine/modules.h
deleted file mode 100644
index 1b161c6..0000000
--- a/ext/kenlm/jam-files/engine/modules.h
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
- */
-#ifndef MODULES_DWA10182001_H
-#define MODULES_DWA10182001_H
-
-#include "lists.h"
-#include "object.h"
-
-typedef struct module_t module_t ;
-struct module_t
-{
- OBJECT * name;
- struct hash * rules;
- struct hash * variables;
- struct hash * variable_indices;
- int num_fixed_variables;
- LIST * * fixed_variables;
- struct hash * imported_modules;
- module_t * class_module;
- struct hash * native_rules;
- int user_module;
-};
-
-module_t * bindmodule( OBJECT * name );
-module_t * root_module();
-void delete_module( module_t * );
-
-void import_module( LIST * module_names, module_t * target_module );
-LIST * imported_modules( module_t * );
-
-struct hash * demand_rules( module_t * );
-
-void module_bind_variables( module_t * );
-
-/*
- * After calling module_add_fixed_var, module_set_fixed_variables must be called
- * before accessing any variables in the module.
- */
-int module_add_fixed_var( module_t *, OBJECT * name, int * n );
-void module_set_fixed_variables( module_t *, int n );
-
-/*
- * Returns the index of the variable or -1 if none exists.
- */
-int module_get_fixed_var( module_t *, OBJECT * name );
-
-void modules_done();
-
-#endif
[47/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/boost-build.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/boost-build.jam b/ext/kenlm/jam-files/boost-build/boost-build.jam
deleted file mode 100644
index 73db049..0000000
--- a/ext/kenlm/jam-files/boost-build/boost-build.jam
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright 2001, 2002 Dave Abrahams
-# Copyright 2002 Rene Rivera
-# Copyright 2003 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-
-boost-build kernel ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/bootstrap.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/bootstrap.jam b/ext/kenlm/jam-files/boost-build/bootstrap.jam
deleted file mode 100644
index af3e8bf..0000000
--- a/ext/kenlm/jam-files/boost-build/bootstrap.jam
+++ /dev/null
@@ -1,18 +0,0 @@
-# Copyright (c) 2003 Vladimir Prus.
-#
-# Use, modification and distribution is subject to the Boost Software
-# License Version 1.0. (See accompanying file LICENSE_1_0.txt or
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# This file handles initial phase of Boost.Build loading.
-# Boost.Jam has already figured out where Boost.Build is
-# and loads this file, which is responsible for initialization
-# of basic facilities such a module system and loading the
-# main Boost.Build module, build-system.jam.
-#
-# Exact operation of this module is not interesting, it makes
-# sense to look at build-system.jam right away.
-
-# Load the kernel/bootstrap.jam, which does all the work.
-.bootstrap-file = $(.bootstrap-file:D)/kernel/bootstrap.jam ;
-include $(.bootstrap-file) ;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/build-system.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/build-system.jam b/ext/kenlm/jam-files/boost-build/build-system.jam
deleted file mode 100644
index 247326a..0000000
--- a/ext/kenlm/jam-files/boost-build/build-system.jam
+++ /dev/null
@@ -1,981 +0,0 @@
-# Copyright 2003, 2005, 2007 Dave Abrahams
-# Copyright 2006, 2007 Rene Rivera
-# Copyright 2003, 2004, 2005, 2006 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# This file is part of Boost Build version 2. You can think of it as forming the
-# main() routine. It is invoked by the bootstrapping code in bootstrap.jam.
-
-import build-request ;
-import builtin ;
-import "class" : new ;
-import configure ;
-import config-cache ;
-import feature ;
-import generators ;
-import make ;
-import modules ;
-import os ;
-import path ;
-import project ;
-import property ;
-import property-set ;
-import regex ;
-import sequence ;
-import targets ;
-import toolset ;
-import utility ;
-import version ;
-import virtual-target ;
-
-
-################################################################################
-#
-# Module global data.
-#
-################################################################################
-
-# Shortcut used in this module for accessing used command-line parameters.
-.argv = [ modules.peek : ARGV ] ;
-
-# Flag indicating we should display additional debugging information related to
-# locating and loading Boost Build configuration files.
-.debug-config = [ MATCH ^(--debug-configuration)$ : $(.argv) ] ;
-
-# Virtual targets obtained when building main targets references on the command
-# line. When running 'bjam --clean main_target' we want to clean only files
-# belonging to that main target so we need to record which targets are produced
-# for it.
-.results-of-main-targets = ;
-
-# Was an XML dump requested?
-.out-xml = [ MATCH ^--out-xml=(.*)$ : $(.argv) ] ;
-
-# Default toolset & version to be used in case no other toolset has been used
-# explicitly by either the loaded configuration files, the loaded project build
-# scripts or an explicit toolset request on the command line. If not specified,
-# an arbitrary default will be used based on the current host OS. This value,
-# while not strictly necessary, has been added to allow testing Boost-Build's
-# default toolset usage functionality.
-.default-toolset = ;
-.default-toolset-version = ;
-
-
-################################################################################
-#
-# Public rules.
-#
-################################################################################
-
-# Returns the property set with the free features from the currently processed
-# build request.
-#
-rule command-line-free-features ( )
-{
- return $(.command-line-free-features) ;
-}
-
-
-# Returns the location of the build system. The primary use case is building
-# Boost where it is sometimes needed to get the location of other components
-# (e.g. BoostBook files) and it is convenient to use locations relative to the
-# Boost Build path.
-#
-rule location ( )
-{
- local r = [ modules.binding build-system ] ;
- return $(r:P) ;
-}
-
-
-# Sets the default toolset & version to be used in case no other toolset has
-# been used explicitly by either the loaded configuration files, the loaded
-# project build scripts or an explicit toolset request on the command line. For
-# more detailed information see the comment related to used global variables.
-#
-rule set-default-toolset ( toolset : version ? )
-{
- .default-toolset = $(toolset) ;
- .default-toolset-version = $(version) ;
-}
-
-rule set-pre-build-hook ( function )
-{
- .pre-build-hook = $(function) ;
-}
-
-rule set-post-build-hook ( function )
-{
- .post-build-hook = $(function) ;
-}
-
-################################################################################
-#
-# Local rules.
-#
-################################################################################
-
-# Returns actual Jam targets to be used for executing a clean request.
-#
-local rule actual-clean-targets ( )
-{
- # The cleaning is tricky. Say, if user says 'bjam --clean foo' where 'foo'
- # is a directory, then we want to clean targets which are in 'foo' as well
- # as those in any children Jamfiles under foo but not in any unrelated
- # Jamfiles. To achieve this we first mark all projects explicitly detected
- # as targets for this build system run as needing to be cleaned.
- for local t in $(targets)
- {
- if [ class.is-a $(t) : project-target ]
- {
- local project = [ $(t).project-module ] ;
- .should-clean-project.$(project) = true ;
- }
- }
-
- # Construct a list of targets explicitly detected on this build system run
- # as a result of building main targets.
- local targets-to-clean ;
- for local t in $(.results-of-main-targets)
- {
- # Do not include roots or sources.
- targets-to-clean += [ virtual-target.traverse $(t) ] ;
- }
- targets-to-clean = [ sequence.unique $(targets-to-clean) ] ;
-
- local to-clean ;
- for local t in [ virtual-target.all-targets ]
- {
- # Remove only derived targets and only those asked to be cleaned,
- # whether directly or by belonging to one of the removed projects.
- local p = [ $(t).project ] ;
- if [ $(t).action ] && ( $(t) in $(targets-to-clean) ||
- [ should-clean-project [ $(p).project-module ] ] )
- {
- to-clean += $(t) ;
- }
- }
-
- local to-clean-actual ;
- for local t in $(to-clean)
- {
- to-clean-actual += [ $(t).actualize ] ;
- }
- return $(to-clean-actual) ;
-}
-
-
-# Given a target id, try to find and return the corresponding target. This is
-# only invoked when there is no Jamfile in ".". This code somewhat duplicates
-# code in project-target.find but we can not reuse that code without a
-# project-targets instance.
-#
-local rule find-target ( target-id )
-{
- local split = [ MATCH (.*)//(.*) : $(target-id) ] ;
-
- local pm ;
- if $(split)
- {
- pm = [ project.find $(split[1]) : "." ] ;
- }
- else
- {
- pm = [ project.find $(target-id) : "." ] ;
- }
-
- local result ;
- if $(pm)
- {
- result = [ project.target $(pm) ] ;
- }
-
- if $(split)
- {
- result = [ $(result).find $(split[2]) ] ;
- }
-
- return $(result) ;
-}
-
-
-# Initializes a new configuration module.
-#
-local rule initialize-config-module ( module-name : location ? )
-{
- project.initialize $(module-name) : $(location) ;
- if USER_MODULE in [ RULENAMES ]
- {
- USER_MODULE $(module-name) ;
- }
-}
-
-
-# Helper rule used to load configuration files. Loads the first configuration
-# file with the given 'filename' at 'path' into module with name 'module-name'.
-# Not finding the requested file may or may not be treated as an error depending
-# on the must-find parameter. Returns a normalized path to the loaded
-# configuration file or nothing if no file was loaded.
-#
-local rule load-config ( module-name : filename : path + : must-find ? )
-{
- if $(.debug-config)
- {
- local path-string = $(path) ;
- if $(path-string) = "" { path-string = . ; }
- ECHO notice: Searching '$(path-string)' for $(module-name)
- configuration file '$(filename)'. ;
- }
- local where = [ GLOB $(path) : $(filename) ] ;
- if $(where)
- {
- where = [ NORMALIZE_PATH $(where[1]) ] ;
- if $(.debug-config)
- {
- local where-string = $(where:D) ;
- if $(where-string) = "" { where-string = . ; }
- where-string = '$(where-string)' ;
- ECHO notice: Loading $(module-name) configuration file '$(filename)'
- from $(where-string:J=" "). ;
- }
-
- # Set source location so that path-constant in config files with
- # relative paths work. This is of most importance for
- # project-config.jam, but may be used in other config files as well.
- local attributes = [ project.attributes $(module-name) ] ;
- $(attributes).set source-location : $(where:D) : exact ;
- modules.load $(module-name) : $(filename) : $(path) ;
- project.load-used-projects $(module-name) ;
- }
- else if $(must-find) || $(.debug-config)
- {
- local path-string = $(path) ;
- if $(path-string) = "" { path-string = . ; }
- path-string = '$(path-string)' ;
- path-string = $(path-string:J=" ") ;
- if $(must-find)
- {
- import errors ;
- errors.user-error Configuration file '$(filename)' not found "in"
- $(path-string). ;
- }
- ECHO notice: Configuration file '$(filename)' not found "in"
- $(path-string). ;
- }
- return $(where) ;
-}
-
-
-# Loads all the configuration files used by Boost Build in the following order:
-#
-# -- test-config --
-# Loaded only if specified on the command-line using the --test-config
-# command-line parameter. It is ok for this file not to exist even if specified.
-# If this configuration file is loaded, regular site and user configuration
-# files will not be. If a relative path is specified, file is searched for in
-# the current folder.
-#
-# -- site-config --
-# Always named site-config.jam. Will only be found if located on the system
-# root path (Windows), /etc (non-Windows), user's home folder or the Boost Build
-# path, in that order. Not loaded in case the test-config configuration file is
-# loaded or the --ignore-site-config command-line option is specified.
-#
-# -- user-config --
-# Named user-config.jam by default or may be named explicitly using the
-# --user-config command-line option or the BOOST_BUILD_USER_CONFIG environment
-# variable. If named explicitly the file is looked for from the current working
-# directory and if the default one is used then it is searched for in the
-# user's home directory and the Boost Build path, in that order. Not loaded in
-# case either the test-config configuration file is loaded or an empty file name
-# is explicitly specified. If the file name has been given explicitly then the
-# file must exist.
-#
-# -- project-config --
-# Always named project-config.jam. Looked up in the current working folder and
-# then upwards through its parents up to the root folder.
-#
-# Test configurations have been added primarily for use by Boost Build's
-# internal unit testing system but may be used freely in other places as well.
-#
-local rule load-configuration-files
-{
- # Flag indicating that site configuration should not be loaded.
- local ignore-site-config =
- [ MATCH ^(--ignore-site-config)$ : $(.argv) ] ;
-
- initialize-config-module test-config ;
- local test-config = [ MATCH ^--test-config=(.*)$ : $(.argv) ] ;
- local uq = [ MATCH \"(.*)\" : $(test-config) ] ;
- if $(uq)
- {
- test-config = $(uq) ;
- }
- if $(test-config)
- {
- local where = [ load-config test-config : $(test-config:BS) :
- $(test-config:D) ] ;
- if $(where)
- {
- if $(.debug-config)
- {
- ECHO "notice: Regular site and user configuration files will" ;
- ECHO "notice: be ignored due to the test configuration being"
- "loaded." ;
- }
- }
- else
- {
- test-config = ;
- }
- }
-
- local user-path = [ os.home-directories ] [ os.environ BOOST_BUILD_PATH ] ;
- local site-path = /etc $(user-path) ;
- if [ os.name ] in NT CYGWIN
- {
- site-path = [ modules.peek : SystemRoot ] $(user-path) ;
- }
-
- if $(.debug-config) && ! $(test-config) && $(ignore-site-config)
- {
- ECHO "notice: Site configuration files will be ignored due to the" ;
- ECHO "notice: --ignore-site-config command-line option." ;
- }
-
- initialize-config-module site-config ;
- if ! $(test-config) && ! $(ignore-site-config)
- {
- load-config site-config : site-config.jam : $(site-path) ;
- }
-
- initialize-config-module user-config ;
- if ! $(test-config)
- {
- local user-config = [ MATCH ^--user-config=(.*)$ : $(.argv) ] ;
- user-config = $(user-config[-1]) ;
- user-config ?= [ os.environ BOOST_BUILD_USER_CONFIG ] ;
- # Special handling for the case when the OS does not strip the quotes
- # around the file name, as is the case when using Cygwin bash.
- user-config = [ utility.unquote $(user-config) ] ;
- local explicitly-requested = $(user-config) ;
- user-config ?= user-config.jam ;
-
- if $(user-config)
- {
- if $(explicitly-requested)
- {
- # Treat explicitly entered user paths as native OS path
- # references and, if non-absolute, root them at the current
- # working directory.
- user-config = [ path.make $(user-config) ] ;
- user-config = [ path.root $(user-config) [ path.pwd ] ] ;
- user-config = [ path.native $(user-config) ] ;
-
- if $(.debug-config)
- {
- ECHO notice: Loading explicitly specified user configuration
- file: ;
- ECHO " $(user-config)" ;
- }
-
- load-config user-config : $(user-config:BS) : $(user-config:D)
- : must-exist ;
- }
- else
- {
- load-config user-config : $(user-config) : $(user-path) ;
- }
- }
- else if $(.debug-config)
- {
- ECHO notice: User configuration file loading explicitly disabled. ;
- }
- }
-
- # We look for project-config.jam from "." upward. I am not sure this is 100%
- # right decision, we might as well check for it only alongside the Jamroot
- # file. However:
- # - We need to load project-config.jam before Jamroot
- # - We probably need to load project-config.jam even if there is no Jamroot
- # - e.g. to implement automake-style out-of-tree builds.
- local file = [ path.glob "." : project-config.jam ] ;
- if ! $(file)
- {
- file = [ path.glob-in-parents "." : project-config.jam ] ;
- }
- if $(file)
- {
- initialize-config-module project-config : $(file:D) ;
- load-config project-config : project-config.jam : $(file:D) ;
- }
-
- project.end-load ;
-}
-
-
-# Autoconfigure toolsets based on any instances of --toolset=xx,yy,...zz or
-# toolset=xx,yy,...zz in the command line. May return additional properties to
-# be processed as if they had been specified by the user.
-#
-local rule process-explicit-toolset-requests
-{
- local extra-properties ;
-
- local option-toolsets = [ regex.split-list [ MATCH ^--toolset=(.*)$ : $(.argv) ] : "," ] ;
- local feature-toolsets = [ regex.split-list [ MATCH ^toolset=(.*)$ : $(.argv) ] : "," ] ;
-
- for local t in $(option-toolsets) $(feature-toolsets)
- {
- # Parse toolset-version/properties.
- local toolset = [ MATCH ([^/]+)/?.* : $(t) ] ;
- local properties = [ feature.expand-subfeatures <toolset>$(toolset) : true ] ;
- local toolset-property = [ property.select <toolset> : $(properties) ] ;
- local known ;
- if $(toolset-property:G=) in [ feature.values <toolset> ]
- {
- known = true ;
- }
-
- # If the toolset is not known, configure it now.
-
- # TODO: we should do 'using $(toolset)' in case no version has been
- # specified and there are no versions defined for the given toolset to
- # allow the toolset to configure its default version. For this we need
- # to know how to detect whether a given toolset has any versions
- # defined. An alternative would be to do this whenever version is not
- # specified but that would require that toolsets correctly handle the
- # case when their default version is configured multiple times which
- # should be checked for all existing toolsets first.
-
- if ! $(known)
- {
- if $(.debug-config)
- {
- ECHO "notice: [cmdline-cfg] toolset $(toolset) not"
- "previously configured; attempting to auto-configure now" ;
- }
- local t,v = [ MATCH ([^-]+)-?(.+)? : $(toolset) ] ;
- toolset.using $(t,v[1]) : $(t,v[2]) ;
- }
-
- # Make sure we get an appropriate property into the build request in
- # case toolset has been specified using the "--toolset=..." command-line
- # option form.
- if ! $(t) in $(.argv) $(feature-toolsets)
- {
- if $(.debug-config)
- {
- ECHO notice: [cmdline-cfg] adding toolset=$(t) to the build
- request. ;
- }
- extra-properties += toolset=$(t) ;
- }
- }
-
- return $(extra-properties) ;
-}
-
-
-# Returns whether the given project (identifed by its project module) should be
-# cleaned because it or any of its parent projects have already been marked as
-# needing to be cleaned in this build. As an optimization, will explicitly mark
-# all encountered project needing to be cleaned in case thay have not already
-# been marked so.
-#
-local rule should-clean-project ( project )
-{
- if ! $(.should-clean-project.$(project))-is-defined
- {
- local r = "" ;
- if ! [ project.is-jamroot-module $(project) ]
- {
- local parent = [ project.attribute $(project) parent-module ] ;
- if $(parent)
- {
- r = [ should-clean-project $(parent) ] ;
- }
- }
- .should-clean-project.$(project) = $(r) ;
- }
-
- return $(.should-clean-project.$(project)) ;
-}
-
-
-################################################################################
-#
-# main()
-# ------
-#
-################################################################################
-
-{
- if --version in $(.argv)
- {
- version.print ;
- EXIT ;
- }
-
- version.verify-engine-version ;
-
- load-configuration-files ;
-
- # Load explicitly specified toolset modules.
- local extra-properties = [ process-explicit-toolset-requests ] ;
-
- # Load the actual project build script modules. We always load the project
- # in the current folder so 'use-project' directives have any chance of being
- # seen. Otherwise, we would not be able to refer to subprojects using target
- # ids.
- local current-project ;
- {
- local current-module = [ project.find "." : "." ] ;
- if $(current-module)
- {
- current-project = [ project.target $(current-module) ] ;
- }
- }
-
- # Load the default toolset module if no other has already been specified.
- if ! [ feature.values <toolset> ]
- {
- local default-toolset = $(.default-toolset) ;
- local default-toolset-version = ;
- if $(default-toolset)
- {
- default-toolset-version = $(.default-toolset-version) ;
- }
- else
- {
- default-toolset = gcc ;
- if [ os.name ] = NT
- {
- default-toolset = msvc ;
- }
- else if [ os.name ] = MACOSX
- {
- default-toolset = darwin ;
- }
- }
-
- ECHO "warning: No toolsets are configured." ;
- ECHO "warning: Configuring default toolset" \"$(default-toolset)\". ;
- ECHO "warning: If the default is wrong, your build may not work correctly." ;
- ECHO "warning: Use the \"toolset=xxxxx\" option to override our guess." ;
- ECHO "warning: For more configuration options, please consult" ;
- ECHO "warning: http://boost.org/boost-build2/doc/html/bbv2/advanced/configuration.html" ;
-
- toolset.using $(default-toolset) : $(default-toolset-version) ;
- }
-
-
- # Parse command line for targets and properties. Note that this requires
- # that all project files already be loaded.
- # FIXME: This is not entirely true. Additional project files may be loaded
- # only later via the project.find() rule when dereferencing encountered
- # target ids containing explicit project references. See what to do about
- # those as such 'lazy loading' may cause problems that are then extremely
- # difficult to debug.
- local build-request = [ build-request.from-command-line $(.argv)
- $(extra-properties) ] ;
- local target-ids = [ $(build-request).get-at 1 ] ;
- local properties = [ $(build-request).get-at 2 ] ;
-
-
- # Expand properties specified on the command line into multiple property
- # sets consisting of all legal property combinations. Each expanded property
- # set will be used for a single build run. E.g. if multiple toolsets are
- # specified then requested targets will be built with each of them.
- if $(properties)
- {
- expanded = [ build-request.expand-no-defaults $(properties) ] ;
- local xexpanded ;
- for local e in $(expanded)
- {
- xexpanded += [ property-set.create [ feature.split $(e) ] ] ;
- }
- expanded = $(xexpanded) ;
- }
- else
- {
- expanded = [ property-set.empty ] ;
- }
-
-
- # Check that we actually found something to build.
- if ! $(current-project) && ! $(target-ids)
- {
- import errors ;
- errors.user-error no Jamfile "in" current directory found, and no target
- references specified. ;
- }
-
-
- # Flags indicating that this build system run has been started in order to
- # clean existing instead of create new targets. Note that these are not the
- # final flag values as they may get changed later on due to some special
- # targets being specified on the command line.
- local clean ; if "--clean" in $(.argv) { clean = true ; }
- local cleanall ; if "--clean-all" in $(.argv) { cleanall = true ; }
-
-
- # List of explicitly requested files to build. Any target references read
- # from the command line parameter not recognized as one of the targets
- # defined in the loaded Jamfiles will be interpreted as an explicitly
- # requested file to build. If any such files are explicitly requested then
- # only those files and the targets they depend on will be built and they
- # will be searched for among targets that would have been built had there
- # been no explicitly requested files.
- local explicitly-requested-files
-
-
- # List of Boost Build meta-targets, virtual-targets and actual Jam targets
- # constructed in this build system run.
- local targets ;
- local virtual-targets ;
- local actual-targets ;
-
-
- # Process each target specified on the command-line and convert it into
- # internal Boost Build target objects. Detect special clean target. If no
- # main Boost Build targets were explictly requested use the current project
- # as the target.
- for local id in $(target-ids)
- {
- if $(id) = clean
- {
- clean = true ;
- }
- else
- {
- local t ;
- if $(current-project)
- {
- t = [ $(current-project).find $(id) : no-error ] ;
- }
- else
- {
- t = [ find-target $(id) ] ;
- }
-
- if ! $(t)
- {
- ECHO "notice: could not find main target" $(id) ;
- ECHO "notice: assuming it is a name of file to create." ;
- explicitly-requested-files += $(id) ;
- }
- else
- {
- targets += $(t) ;
- }
- }
- }
- if ! $(targets)
- {
- targets += [ project.target [ project.module-name "." ] ] ;
- }
-
- if [ option.get dump-generators : : true ]
- {
- generators.dump ;
- }
-
- # We wish to put config.log in the build directory corresponding to Jamroot,
- # so that the location does not differ depending on the directory we run the
- # build from. The amount of indirection necessary here is scary.
- local first-project = [ $(targets[0]).project ] ;
- local first-project-root-location = [ $(first-project).get project-root ] ;
- local first-project-root-module = [ project.load
- $(first-project-root-location) ] ;
- local first-project-root = [ project.target $(first-project-root-module) ] ;
- local first-build-build-dir = [ $(first-project-root).build-dir ] ;
- configure.set-log-file $(first-build-build-dir)/config.log ;
- config-cache.load $(first-build-build-dir)/project-cache.jam ;
-
- # Now that we have a set of targets to build and a set of property sets to
- # build the targets with, we can start the main build process by using each
- # property set to generate virtual targets from all of our listed targets
- # and any of their dependants.
- for local p in $(expanded)
- {
- .command-line-free-features = [ property-set.create [ $(p).free ] ] ;
- for local t in $(targets)
- {
- local g = [ $(t).generate $(p) ] ;
- if ! [ class.is-a $(t) : project-target ]
- {
- .results-of-main-targets += $(g[2-]) ;
- }
- virtual-targets += $(g[2-]) ;
- }
- }
-
-
- # Convert collected virtual targets into actual raw Jam targets.
- for t in $(virtual-targets)
- {
- actual-targets += [ $(t).actualize ] ;
- }
-
- config-cache.save ;
-
-
- # If XML data output has been requested prepare additional rules and targets
- # so we can hook into Jam to collect build data while its building and have
- # it trigger the final XML report generation after all the planned targets
- # have been built.
- if $(.out-xml)
- {
- # Get a qualified virtual target name.
- rule full-target-name ( target )
- {
- local name = [ $(target).name ] ;
- local project = [ $(target).project ] ;
- local project-path = [ $(project).get location ] ;
- return $(project-path)//$(name) ;
- }
-
- # Generate an XML file containing build statistics for each constituent.
- #
- rule out-xml ( xml-file : constituents * )
- {
- # Prepare valid XML header and footer with some basic info.
- local nl = "
-" ;
- local os = [ modules.peek : OS OSPLAT JAMUNAME ] "" ;
- local timestamp = [ modules.peek : JAMDATE ] ;
- local cwd = [ PWD ] ;
- local command = $(.argv) ;
- local bb-version = [ version.boost-build ] ;
- .header on $(xml-file) =
- "<?xml version=\"1.0\" encoding=\"utf-8\"?>"
- "$(nl)<build format=\"1.0\" version=\"$(bb-version)\">"
- "$(nl) <os name=\"$(os[1])\" platform=\"$(os[2])\"><![CDATA[$(os[3-]:J= )]]></os>"
- "$(nl) <timestamp><![CDATA[$(timestamp)]]></timestamp>"
- "$(nl) <directory><![CDATA[$(cwd)]]></directory>"
- "$(nl) <command><![CDATA[\"$(command:J=\" \")\"]]></command>"
- ;
- .footer on $(xml-file) =
- "$(nl)</build>" ;
-
- # Generate the target dependency graph.
- .contents on $(xml-file) +=
- "$(nl) <targets>" ;
- for local t in [ virtual-target.all-targets ]
- {
- local action = [ $(t).action ] ;
- if $(action)
- # If a target has no action, it has no dependencies.
- {
- local name = [ full-target-name $(t) ] ;
- local sources = [ $(action).sources ] ;
- local dependencies ;
- for local s in $(sources)
- {
- dependencies += [ full-target-name $(s) ] ;
- }
-
- local path = [ $(t).path ] ;
- local jam-target = [ $(t).actual-name ] ;
-
- .contents on $(xml-file) +=
- "$(nl) <target>"
- "$(nl) <name><![CDATA[$(name)]]></name>"
- "$(nl) <dependencies>"
- "$(nl) <dependency><![CDATA[$(dependencies)]]></dependency>"
- "$(nl) </dependencies>"
- "$(nl) <path><![CDATA[$(path)]]></path>"
- "$(nl) <jam-target><![CDATA[$(jam-target)]]></jam-target>"
- "$(nl) </target>"
- ;
- }
- }
- .contents on $(xml-file) +=
- "$(nl) </targets>" ;
-
- # Build $(xml-file) after $(constituents). Do so even if a
- # constituent action fails and regenerate the xml on every bjam run.
- INCLUDES $(xml-file) : $(constituents) ;
- ALWAYS $(xml-file) ;
- __ACTION_RULE__ on $(xml-file) =
- build-system.out-xml.generate-action ;
- out-xml.generate $(xml-file) ;
- }
-
- # The actual build actions are here; if we did this work in the actions
- # clause we would have to form a valid command line containing the
- # result of @(...) below (the name of the XML file).
- #
- rule out-xml.generate-action ( args * : xml-file
- : command status start end user system : output ? )
- {
- local contents =
- [ on $(xml-file) return $(.header) $(.contents) $(.footer) ] ;
- local f = @($(xml-file):E=$(contents)) ;
- }
-
- # Nothing to do here; the *real* actions happen in
- # out-xml.generate-action.
- actions quietly out-xml.generate { }
-
- # Define the out-xml file target, which depends on all the targets so
- # that it runs the collection after the targets have run.
- out-xml $(.out-xml) : $(actual-targets) ;
-
- # Set up a global __ACTION_RULE__ that records all the available
- # statistics about each actual target in a variable "on" the --out-xml
- # target.
- #
- rule out-xml.collect ( xml-file : target : command status start end user
- system : output ? )
- {
- local nl = "
-" ;
- # Open the action with some basic info.
- .contents on $(xml-file) +=
- "$(nl) <action status=\"$(status)\" start=\"$(start)\" end=\"$(end)\" user=\"$(user)\" system=\"$(system)\">" ;
-
- # If we have an action object we can print out more detailed info.
- local action = [ on $(target) return $(.action) ] ;
- if $(action)
- {
- local action-name = [ $(action).action-name ] ;
- local action-sources = [ $(action).sources ] ;
- local action-props = [ $(action).properties ] ;
-
- # The qualified name of the action which we created the target.
- .contents on $(xml-file) +=
- "$(nl) <name><![CDATA[$(action-name)]]></name>" ;
-
- # The sources that made up the target.
- .contents on $(xml-file) +=
- "$(nl) <sources>" ;
- for local source in $(action-sources)
- {
- local source-actual = [ $(source).actual-name ] ;
- .contents on $(xml-file) +=
- "$(nl) <source><![CDATA[$(source-actual)]]></source>" ;
- }
- .contents on $(xml-file) +=
- "$(nl) </sources>" ;
-
- # The properties that define the conditions under which the
- # target was built.
- .contents on $(xml-file) +=
- "$(nl) <properties>" ;
- for local prop in [ $(action-props).raw ]
- {
- local prop-name = [ MATCH ^<(.*)>$ : $(prop:G) ] ;
- .contents on $(xml-file) +=
- "$(nl) <property name=\"$(prop-name)\"><![CDATA[$(prop:G=)]]></property>" ;
- }
- .contents on $(xml-file) +=
- "$(nl) </properties>" ;
- }
-
- local locate = [ on $(target) return $(LOCATE) ] ;
- locate ?= "" ;
- .contents on $(xml-file) +=
- "$(nl) <jam-target><![CDATA[$(target)]]></jam-target>"
- "$(nl) <path><![CDATA[$(target:G=:R=$(locate))]]></path>"
- "$(nl) <command><![CDATA[$(command)]]></command>"
- "$(nl) <output><![CDATA[$(output)]]></output>" ;
- .contents on $(xml-file) +=
- "$(nl) </action>" ;
- }
-
- # When no __ACTION_RULE__ is set "on" a target, the search falls back to
- # the global module.
- module
- {
- __ACTION_RULE__ = build-system.out-xml.collect
- [ modules.peek build-system : .out-xml ] ;
- }
-
- IMPORT
- build-system :
- out-xml.collect
- out-xml.generate-action
- : :
- build-system.out-xml.collect
- build-system.out-xml.generate-action
- ;
- }
-
- local j = [ option.get jobs ] ;
- if $(j)
- {
- modules.poke : PARALLELISM : $(j) ;
- }
-
- local k = [ option.get keep-going : true : true ] ;
- if $(k) in "on" "yes" "true"
- {
- modules.poke : KEEP_GOING : 1 ;
- }
- else if $(k) in "off" "no" "false"
- {
- modules.poke : KEEP_GOING : 0 ;
- }
- else
- {
- EXIT "error: Invalid value for the --keep-going option" ;
- }
-
- # The 'all' pseudo target is not strictly needed expect in the case when we
- # use it below but people often assume they always have this target
- # available and do not declare it themselves before use which may cause
- # build failures with an error message about not being able to build the
- # 'all' target.
- NOTFILE all ;
-
- # And now that all the actual raw Jam targets and all the dependencies
- # between them have been prepared all that is left is to tell Jam to update
- # those targets.
- if $(explicitly-requested-files)
- {
- # Note that this case can not be joined with the regular one when only
- # exact Boost Build targets are requested as here we do not build those
- # requested targets but only use them to construct the dependency tree
- # needed to build the explicitly requested files.
- UPDATE $(explicitly-requested-files:G=e) $(.out-xml) ;
- }
- else if $(cleanall)
- {
- UPDATE clean-all ;
- }
- else if $(clean)
- {
- common.Clean clean : [ actual-clean-targets ] ;
- UPDATE clean ;
- }
- else
- {
- configure.print-configure-checks-summary ;
-
- if $(.pre-build-hook)
- {
- $(.pre-build-hook) ;
- }
-
- DEPENDS all : $(actual-targets) ;
- if UPDATE_NOW in [ RULENAMES ]
- {
- local ok = [ UPDATE_NOW all $(.out-xml) ] ;
- if $(.post-build-hook)
- {
- $(.post-build-hook) $(ok) ;
- }
- # Prevent automatic update of the 'all' target, now that we have
- # explicitly updated what we wanted.
- UPDATE ;
- }
- else
- {
- UPDATE all $(.out-xml) ;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/build/ac.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/build/ac.jam b/ext/kenlm/jam-files/boost-build/build/ac.jam
deleted file mode 100644
index 71bc16c..0000000
--- a/ext/kenlm/jam-files/boost-build/build/ac.jam
+++ /dev/null
@@ -1,303 +0,0 @@
-# Copyright (c) 2010 Vladimir Prus.
-# Copyright (c) 2013 Steven Watanabe
-#
-# Use, modification and distribution is subject to the Boost Software
-# License Version 1.0. (See accompanying file LICENSE_1_0.txt or
-# http://www.boost.org/LICENSE_1_0.txt)
-
-import property-set ;
-import path ;
-import modules ;
-import "class" ;
-import errors ;
-import configure ;
-import project ;
-import virtual-target ;
-import generators ;
-import property ;
-import print ;
-
-project.initialize $(__name__) ;
-.project = [ project.current ] ;
-project ac ;
-
-rule generate-include ( target : sources * : properties * )
-{
- local header = [ property.select <include> : $(properties) ] ;
- print.output $(target) ;
- print.text "#include <$(header:G=)>" : true ;
-}
-
-rule generate-main ( target : sources * : properties * )
-{
- print.output $(target) ;
- print.text "int main() {}" : true ;
-}
-
-rule find-include-path ( properties : header : provided-path ? )
-{
- if $(provided-path) && [ path.exists [ path.root $(header) $(provided-path) ] ]
- {
- return $(provided-path) ;
- }
- else
- {
- local a = [ class.new action : ac.generate-include : [ property-set.create <include>$(header) ] ] ;
- local cpp = [ class.new file-target $(header).cpp exact : CPP : $(.project) : $(a) ] ;
- cpp = [ virtual-target.register $(cpp) ] ;
- local result = [ generators.construct $(.project) $(header) : OBJ : $(properties) : $(cpp) : true ] ;
- local jam-targets ;
- for t in $(result[2-])
- {
- jam-targets += [ $(t).actualize ] ;
- }
- if [ UPDATE_NOW $(jam-targets) : [ modules.peek configure : .log-fd ]
- : ignore-minus-n : ignore-minus-q ]
- {
- return %default ;
- }
- }
-}
-
-rule construct-library ( name : property-set : provided-path ? )
-{
- property-set = [ $(property-set).refine [ property-set.create $(link-opt) ] ] ;
- local lib-props = [ $(property-set).add-raw <name>$(name) <search>$(provided-path) ] ;
- return [ generators.construct $(.project) lib-$(name)
- : SEARCHED_LIB : $(lib-props) : : true ] ;
-}
-
-
-rule find-library ( properties : names + : provided-path ? )
-{
- local result ;
- if ! $(.main.cpp)
- {
- local a = [ class.new action : ac.generate-main :
- [ property-set.empty ] ] ;
- .main.cpp = [ virtual-target.register
- [ class.new file-target main.cpp exact
- : CPP : $(.project) : $(a) ] ] ;
- }
- if [ $(properties).get <link> ] = shared
- {
- link-opts = <link>shared <link>static ;
- }
- else
- {
- link-opts = <link>static <link>shared ;
- }
- while $(link-opts)
- {
- local names-iter = $(names) ;
- properties = [ $(properties).refine [ property-set.create $(link-opts[1]) ] ] ;
- while $(names-iter)
- {
- local name = $(names-iter[1]) ;
- local lib = [ construct-library $(name) : $(properties) : $(provided-path) ] ;
- local test = [ generators.construct $(.project) $(name) : EXE
- : [ $(properties).add $(lib[1]) ] : $(.main.cpp) $(lib[2-])
- : true ] ;
- local jam-targets ;
- for t in $(test[2-])
- {
- jam-targets += [ $(t).actualize ] ;
- }
- if [ UPDATE_NOW $(jam-targets) : [ modules.peek configure : .log-fd ]
- : ignore-minus-n : ignore-minus-q ]
- {
- result = $(name) $(link-opts[1]) ;
- names-iter = ; link-opts = ; # break
- }
- names-iter = $(names-iter[2-]) ;
- }
- link-opts = $(link-opts[2-]) ;
- }
- return $(result) ;
-}
-
-class ac-library : basic-target
-{
- import errors ;
- import indirect ;
- import virtual-target ;
- import ac ;
- import configure ;
- import config-cache ;
-
- rule __init__ ( name : project : requirements * : include-path ? : library-path ? : library-name ? )
- {
- basic-target.__init__ $(name) : $(project) : : $(requirements) ;
-
- reconfigure $(include-path) : $(library-path) : $(library-name) ;
- }
-
- rule set-header ( header )
- {
- self.header = $(header) ;
- }
-
- rule set-default-names ( names + )
- {
- self.default-names = $(names) ;
- }
-
- rule reconfigure ( include-path ? : library-path ? : library-name ? )
- {
- if $(include-path) || $(library-path) || $(library-name)
- {
- check-not-configured ;
-
- self.include-path = $(include-path) ;
- self.library-path = $(library-path) ;
- self.library-name = $(library-name) ;
- }
- }
-
- rule set-target ( target )
- {
- check-not-configured ;
- self.target = $(target) ;
- }
-
- rule check-not-configured ( )
- {
- if $(self.include-path) || $(self.library-path) || $(self.library-name) || $(self.target)
- {
- errors.user-error [ name ] "is already configured" ;
- }
- }
-
- rule construct ( name : sources * : property-set )
- {
- if $(self.target)
- {
- return [ $(self.target).generate $(property-set) ] ;
- }
- else
- {
- local use-environment ;
- if ! $(self.library-name) && ! $(self.include-path) && ! $(self.library-path)
- {
- use-environment = true ;
- }
- local libnames = $(self.library-name) ;
- if ! $(libnames) && $(use-environment)
- {
- libnames = [ modules.peek : $(name:U)_NAME ] ;
- # Backward compatibility only.
- libnames ?= [ modules.peek : $(name:U)_BINARY ] ;
- }
- libnames ?= $(self.default-names) ;
-
- local include-path = $(self.include-path) ;
- if ! $(include-path) && $(use-environment)
- {
- include-path = [ modules.peek : $(name:U)_INCLUDE ] ;
- }
-
- local library-path = $(self.library-path) ;
- if ! $(library-path) && $(use-environment)
- {
- library-path = [ modules.peek : $(name:U)_LIBRARY_PATH ] ;
- # Backwards compatibility only
- library-path ?= [ modules.peek : $(name:U)_LIBPATH ] ;
- }
-
- local toolset = [ $(property-set).get <toolset> ] ;
- local toolset-version-property = "<toolset-$(toolset):version>" ;
- local relevant = [ property.select <target-os> <toolset>
- $(toolset-version-property) <link> <address-model> <architecture> :
- [ $(property-set).raw ] ] ;
-
- local key = ac-library-$(name)-$(relevant:J=-) ;
- local lookup = [ config-cache.get $(key) ] ;
-
- if $(lookup)
- {
- if $(lookup) = missing
- {
- configure.log-library-search-result $(name) : "no (cached)" ;
- return [ property-set.empty ] ;
- }
- else
- {
- local includes = $(lookup[1]) ;
- if $(includes) = %default
- {
- includes = ;
- }
- local library = [ ac.construct-library $(lookup[2]) :
- [ $(property-set).refine [ property-set.create $(lookup[3]) ] ] : $(library-path) ] ;
- configure.log-library-search-result $(name) : "yes (cached)" ;
- return [ $(library[1]).add-raw <include>$(includes) ] $(library[2-]) ;
- }
- }
- else
- {
- local includes = [ ac.find-include-path $(property-set) : $(self.header) : $(include-path) ] ;
- local library = [ ac.find-library $(property-set) : $(libnames) : $(library-path) ] ;
- if $(includes) && $(library)
- {
- config-cache.set $(key) : $(includes) $(library) ;
- if $(includes) = %default
- {
- includes = ;
- }
- library = [ ac.construct-library $(library[1]) :
- [ $(property-set).refine [ property-set.create $(library[2]) ] ] : $(library-path) ] ;
- configure.log-library-search-result $(name) : "yes" ;
- return [ $(library[1]).add-raw <include>$(includes) ] $(library[2-]) ;
- }
- else
- {
- config-cache.set $(key) : missing ;
- configure.log-library-search-result $(name) : "no" ;
- return [ property-set.empty ] ;
- }
- }
- }
- }
-}
-
-class check-library-worker
-{
- import property-set ;
- import targets ;
- import property ;
-
- rule __init__ ( target : true-properties * : false-properties * )
- {
- self.target = $(target) ;
- self.true-properties = $(true-properties) ;
- self.false-properties = $(false-properties) ;
- }
-
- rule check ( properties * )
- {
- local choosen ;
- local t = [ targets.current ] ;
- local p = [ $(t).project ] ;
- local ps = [ property-set.create $(properties) ] ;
- ps = [ $(ps).propagated ] ;
- local generated =
- [ targets.generate-from-reference $(self.target) : $(p) : $(ps) ] ;
- if $(generated[2])
- {
- choosen = $(self.true-properties) ;
- }
- else
- {
- choosen = $(self.false-properties) ;
- }
- return [ property.evaluate-conditionals-in-context $(choosen) :
- $(properties) ] ;
- }
-}
-
-rule check-library ( target : true-properties * : false-properties * )
-{
- local instance = [ class.new check-library-worker $(target) :
- $(true-properties) : $(false-properties) ] ;
- return <conditional>@$(instance).check ;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/build/alias.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/build/alias.jam b/ext/kenlm/jam-files/boost-build/build/alias.jam
deleted file mode 100644
index 9ac8cb8..0000000
--- a/ext/kenlm/jam-files/boost-build/build/alias.jam
+++ /dev/null
@@ -1,74 +0,0 @@
-# Copyright 2003, 2004, 2006 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# This module defines the 'alias' rule and the associated target class.
-#
-# Alias is just a main target which returns its source targets without any
-# processing. For example:
-#
-# alias bin : hello test_hello ;
-# alias lib : helpers xml_parser ;
-#
-# Another important use of 'alias' is to conveniently group source files:
-#
-# alias platform-src : win.cpp : <os>NT ;
-# alias platform-src : linux.cpp : <os>LINUX ;
-# exe main : main.cpp platform-src ;
-#
-# Lastly, it is possible to create a local alias for some target, with different
-# properties:
-#
-# alias big_lib : : @/external_project/big_lib/<link>static ;
-#
-
-import "class" : new ;
-import project ;
-import property-set ;
-import targets ;
-
-
-class alias-target-class : basic-target
-{
- rule __init__ ( name : project : sources * : requirements *
- : default-build * : usage-requirements * )
- {
- basic-target.__init__ $(name) : $(project) : $(sources) :
- $(requirements) : $(default-build) : $(usage-requirements) ;
- }
-
- rule construct ( name : source-targets * : property-set )
- {
- return [ property-set.empty ] $(source-targets) ;
- }
-
- rule compute-usage-requirements ( subvariant )
- {
- local base = [ basic-target.compute-usage-requirements $(subvariant) ] ;
- return [ $(base).add [ $(subvariant).sources-usage-requirements ] ] ;
- }
-}
-
-
-# Declares the 'alias' target. It will process its sources virtual-targets by
-# returning them unaltered as its own constructed virtual-targets.
-#
-rule alias ( name : sources * : requirements * : default-build * :
- usage-requirements * )
-{
- local project = [ project.current ] ;
-
- targets.main-target-alternative
- [ new alias-target-class $(name) : $(project)
- : [ targets.main-target-sources $(sources) : $(name) : no-renaming ]
- : [ targets.main-target-requirements $(requirements) : $(project) ]
- : [ targets.main-target-default-build $(default-build) : $(project)
- ]
- : [ targets.main-target-usage-requirements $(usage-requirements) :
- $(project) ]
- ] ;
-}
-
-
-IMPORT $(__name__) : alias : : alias ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/build/build-request.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/build/build-request.jam b/ext/kenlm/jam-files/boost-build/build/build-request.jam
deleted file mode 100644
index 2a1bbb4..0000000
--- a/ext/kenlm/jam-files/boost-build/build/build-request.jam
+++ /dev/null
@@ -1,322 +0,0 @@
-# Copyright 2002 Dave Abrahams
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-import "class" : new ;
-import sequence ;
-import set ;
-import regex ;
-import feature ;
-import property ;
-import container ;
-import string ;
-
-
-# Transform property-set by applying f to each component property.
-#
-local rule apply-to-property-set ( f property-set )
-{
- local properties = [ feature.split $(property-set) ] ;
- return [ string.join [ $(f) $(properties) ] : / ] ;
-}
-
-
-# Expand the given build request by combining all property-sets which do not
-# specify conflicting non-free features. Expects all the project files to
-# already be loaded.
-#
-rule expand-no-defaults ( property-sets * )
-{
- # First make all features and subfeatures explicit.
- local expanded-property-sets = [ sequence.transform apply-to-property-set
- feature.expand-subfeatures : $(property-sets) ] ;
-
- # Now combine all of the expanded property-sets
- local product = [ x-product $(expanded-property-sets) : $(feature-space) ] ;
-
- return $(product) ;
-}
-
-
-# Implementation of x-product, below. Expects all the project files to already
-# be loaded.
-#
-local rule x-product-aux ( property-sets + )
-{
- local result ;
- local p = [ feature.split $(property-sets[1]) ] ;
- local f = [ set.difference $(p:G) : [ feature.free-features ] ] ;
- local seen ;
- # No conflict with things used at a higher level?
- if ! [ set.intersection $(f) : $(x-product-used) ]
- {
- local x-product-seen ;
- {
- # Do not mix in any conflicting features.
- local x-product-used = $(x-product-used) $(f) ;
-
- if $(property-sets[2])
- {
- local rest = [ x-product-aux $(property-sets[2-]) : $(feature-space) ] ;
- result = $(property-sets[1])/$(rest) ;
- }
-
- result ?= $(property-sets[1]) ;
- }
-
- # If we did not encounter a conflicting feature lower down, do not
- # recurse again.
- if ! [ set.intersection $(f) : $(x-product-seen) ]
- {
- property-sets = ;
- }
-
- seen = $(x-product-seen) ;
- }
-
- if $(property-sets[2])
- {
- result += [ x-product-aux $(property-sets[2-]) : $(feature-space) ] ;
- }
-
- # Note that we have seen these features so that higher levels will recurse
- # again without them set.
- x-product-seen += $(f) $(seen) ;
- return $(result) ;
-}
-
-
-# Return the cross-product of all elements of property-sets, less any that would
-# contain conflicting values for single-valued features. Expects all the project
-# files to already be loaded.
-#
-local rule x-product ( property-sets * )
-{
- if $(property-sets).non-empty
- {
- # Prepare some "scoped globals" that can be used by the implementation
- # function, x-product-aux.
- local x-product-seen x-product-used ;
- return [ x-product-aux $(property-sets) : $(feature-space) ] ;
- }
- # Otherwise return empty.
-}
-
-
-# Returns true if either 'v' or the part of 'v' before the first '-' symbol is
-# an implicit value. Expects all the project files to already be loaded.
-#
-local rule looks-like-implicit-value ( v )
-{
- if [ feature.is-implicit-value $(v) ]
- {
- return true ;
- }
- else
- {
- local split = [ regex.split $(v) - ] ;
- if [ feature.is-implicit-value $(split[1]) ]
- {
- return true ;
- }
- }
-}
-
-
-# Takes the command line tokens (such as taken from the ARGV rule) and
-# constructs a build request from them. Returns a vector of two vectors (where
-# "vector" means container.jam's "vector"). First is the set of targets
-# specified in the command line, and second is the set of requested build
-# properties. Expects all the project files to already be loaded.
-#
-rule from-command-line ( command-line * )
-{
- local targets ;
- local properties ;
-
- command-line = $(command-line[2-]) ;
- local skip-next = ;
- for local e in $(command-line)
- {
- if $(skip-next)
- {
- skip-next = ;
- }
- else if ! [ MATCH ^(-) : $(e) ]
- {
- # Build request spec either has "=" in it or completely consists of
- # implicit feature values.
- local fs = feature-space ;
- if [ MATCH "(.*=.*)" : $(e) ]
- || [ looks-like-implicit-value $(e:D=) : $(feature-space) ]
- {
- properties += [ convert-command-line-element $(e) :
- $(feature-space) ] ;
- }
- else if $(e)
- {
- targets += $(e) ;
- }
- }
- else if [ MATCH "^(-[-ldjfsto])$" : $(e) ]
- {
- skip-next = true ;
- }
- }
- return [ new vector
- [ new vector $(targets) ]
- [ new vector $(properties) ] ] ;
-}
-
-
-# Converts one element of command line build request specification into internal
-# form. Expects all the project files to already be loaded.
-#
-local rule convert-command-line-element ( e )
-{
- local result ;
- local parts = [ regex.split $(e) "/" ] ;
- while $(parts)
- {
- local p = $(parts[1]) ;
- local m = [ MATCH "([^=]*)=(.*)" : $(p) ] ;
- local lresult ;
- local feature ;
- local values ;
- if $(m)
- {
- feature = $(m[1]) ;
- values = [ regex.split $(m[2]) "," ] ;
- lresult = <$(feature)>$(values) ;
- }
- else
- {
- lresult = [ regex.split $(p) "," ] ;
- }
-
- if $(feature) && free in [ feature.attributes <$(feature)> ]
- {
- # If we have free feature, then the value is everything
- # until the end of the command line token. Slashes in
- # the following string are not taked to mean separation
- # of properties. Commas are also not interpreted specially.
- values = $(values:J=,) ;
- values = $(values) $(parts[2-]) ;
- values = $(values:J=/) ;
- lresult = <$(feature)>$(values) ;
- parts = ;
- }
-
- if ! [ MATCH (.*-.*) : $(p) ]
- {
- # property.validate cannot handle subfeatures, so we avoid the check
- # here.
- for local p in $(lresult)
- {
- property.validate $(p) : $(feature-space) ;
- }
- }
-
- if ! $(result)
- {
- result = $(lresult) ;
- }
- else
- {
- result = $(result)/$(lresult) ;
- }
-
- parts = $(parts[2-]) ;
- }
-
- return $(result) ;
-}
-
-
-rule __test__ ( )
-{
- import assert ;
- import feature ;
-
- feature.prepare-test build-request-test-temp ;
-
- import build-request ;
- import build-request : expand-no-defaults : build-request.expand-no-defaults ;
- import errors : try catch ;
- import feature : feature subfeature ;
-
- feature toolset : gcc msvc borland : implicit ;
- subfeature toolset gcc : version : 2.95.2 2.95.3 2.95.4
- 3.0 3.0.1 3.0.2 : optional ;
-
- feature variant : debug release : implicit composite ;
- feature inlining : on off ;
- feature "include" : : free ;
-
- feature stdlib : native stlport : implicit ;
-
- feature runtime-link : dynamic static : symmetric ;
-
- # Empty build requests should expand to empty.
- assert.result
- : build-request.expand-no-defaults ;
-
- assert.result
- <toolset>gcc/<toolset-gcc:version>3.0.1/<stdlib>stlport/<variant>debug
- <toolset>msvc/<stdlib>stlport/<variant>debug
- <toolset>msvc/<variant>debug
- : build-request.expand-no-defaults gcc-3.0.1/stlport msvc/stlport msvc debug ;
-
- assert.result
- <toolset>gcc/<toolset-gcc:version>3.0.1/<stdlib>stlport/<variant>debug
- <toolset>msvc/<variant>debug
- <variant>debug/<toolset>msvc/<stdlib>stlport
- : build-request.expand-no-defaults gcc-3.0.1/stlport msvc debug msvc/stlport ;
-
- assert.result
- <toolset>gcc/<toolset-gcc:version>3.0.1/<stdlib>stlport/<variant>debug/<inlining>off
- <toolset>gcc/<toolset-gcc:version>3.0.1/<stdlib>stlport/<variant>release/<inlining>off
- : build-request.expand-no-defaults gcc-3.0.1/stlport debug release <inlining>off ;
-
- assert.result
- <include>a/b/c/<toolset>gcc/<toolset-gcc:version>3.0.1/<stdlib>stlport/<variant>debug/<include>x/y/z
- <include>a/b/c/<toolset>msvc/<stdlib>stlport/<variant>debug/<include>x/y/z
- <include>a/b/c/<toolset>msvc/<variant>debug/<include>x/y/z
- : build-request.expand-no-defaults <include>a/b/c gcc-3.0.1/stlport msvc/stlport msvc debug <include>x/y/z ;
-
- local r ;
-
- r = [ build-request.from-command-line bjam debug runtime-link=dynamic ] ;
- assert.equal [ $(r).get-at 1 ] : ;
- assert.equal [ $(r).get-at 2 ] : debug <runtime-link>dynamic ;
-
- try ;
- {
- build-request.from-command-line bjam gcc/debug runtime-link=dynamic/static ;
- }
- catch \"static\" is not an implicit feature value ;
-
- r = [ build-request.from-command-line bjam -d2 --debug debug target runtime-link=dynamic ] ;
- assert.equal [ $(r).get-at 1 ] : target ;
- assert.equal [ $(r).get-at 2 ] : debug <runtime-link>dynamic ;
-
- r = [ build-request.from-command-line bjam debug runtime-link=dynamic,static ] ;
- assert.equal [ $(r).get-at 1 ] : ;
- assert.equal [ $(r).get-at 2 ] : debug <runtime-link>dynamic <runtime-link>static ;
-
- r = [ build-request.from-command-line bjam debug gcc/runtime-link=dynamic,static ] ;
- assert.equal [ $(r).get-at 1 ] : ;
- assert.equal [ $(r).get-at 2 ] : debug gcc/<runtime-link>dynamic
- gcc/<runtime-link>static ;
-
- r = [ build-request.from-command-line bjam msvc gcc,borland/runtime-link=static ] ;
- assert.equal [ $(r).get-at 1 ] : ;
- assert.equal [ $(r).get-at 2 ] : msvc gcc/<runtime-link>static
- borland/<runtime-link>static ;
-
- r = [ build-request.from-command-line bjam gcc-3.0 ] ;
- assert.equal [ $(r).get-at 1 ] : ;
- assert.equal [ $(r).get-at 2 ] : gcc-3.0 ;
-
- feature.finish-test build-request-test-temp ;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/build/config-cache.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/build/config-cache.jam b/ext/kenlm/jam-files/boost-build/build/config-cache.jam
deleted file mode 100644
index 5297dbb..0000000
--- a/ext/kenlm/jam-files/boost-build/build/config-cache.jam
+++ /dev/null
@@ -1,64 +0,0 @@
-# Copyright 2012 Steven Watanabe
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-import modules ;
-import errors ;
-import regex ;
-import path ;
-import project ;
-
-rule get ( name )
-{
- return $(.vars.$(name)) ;
-}
-
-rule set ( name : value * )
-{
- .all-vars += $(name) ;
- .vars.$(name) = $(value) ;
-}
-
-rule save ( )
-{
- if $(.cache-file)
- {
- local cache-file-native = [ path.native $(.cache-file) ] ;
- local target = <new-cache-file>$(cache-file-native) ;
- local contents = "# Automatically generated by Boost.Build.\n# Do not edit.\n\nmodule config-cache {\n" ;
- for local var in $(.all-vars)
- {
- local transformed ;
- for local value in $(.vars.$(var))
- {
- transformed += [ regex.escape $(value) : \"\\ : \\ ] ;
- }
- local quoted = \"$(transformed)\" ;
- contents += " set \"$(var)\" : $(quoted:J= ) ;\n" ;
- }
- contents += "}\n" ;
- FILE_CONTENTS on $(target) = $(contents) ;
- ALWAYS $(target) ;
- config-cache.write $(target) ;
- UPDATE_NOW $(target) : [ modules.peek configure : .log-fd ] : ignore-minus-n ;
- }
-}
-
-actions write
-{
- @($(STDOUT):E=$(FILE_CONTENTS:J=)) > "$(<)"
-}
-
-rule load ( cache-file )
-{
- if $(.cache-file)
- {
- errors.error duplicate load of cache file ;
- }
- cache-file = [ path.native $(cache-file) ] ;
- if [ path.exists $(cache-file) ] && ! ( --reconfigure in [ modules.peek : ARGV ] )
- {
- include <old-cache-file>$(cache-file) ;
- }
- .cache-file = $(cache-file) ;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/build/configure.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/build/configure.jam b/ext/kenlm/jam-files/boost-build/build/configure.jam
deleted file mode 100644
index 66b81b3..0000000
--- a/ext/kenlm/jam-files/boost-build/build/configure.jam
+++ /dev/null
@@ -1,281 +0,0 @@
-# Copyright (c) 2010 Vladimir Prus.
-#
-# Use, modification and distribution is subject to the Boost Software
-# License Version 1.0. (See accompanying file LICENSE_1_0.txt or
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# This module defines function to help with two main tasks:
-#
-# - Discovering build-time configuration for the purposes of adjusting the build
-# process.
-# - Reporting what is built, and how it is configured.
-
-import "class" : new ;
-import common ;
-import path ;
-import property ;
-import property-set ;
-import targets ;
-import config-cache ;
-
-
-rule log-summary ( )
-{
-}
-
-
-.width = 30 ;
-
-rule set-width ( width )
-{
- .width = $(width) ;
-}
-
-
-# Declare that the components specified by the parameter exist.
-#
-rule register-components ( components * )
-{
- .components += $(components) ;
-}
-
-
-# Declare that the components specified by the parameters will be built.
-#
-rule components-building ( components * )
-{
- .built-components += $(components) ;
-}
-
-
-# Report something about component configuration that the user should better
-# know.
-#
-rule log-component-configuration ( component : message )
-{
- # FIXME: Implement per-property-set logs.
- .component-logs.$(component) += $(message) ;
-}
-
-
-rule log-check-result ( result )
-{
- if ! $(.announced-checks)
- {
- ECHO "Performing configuration checks\n" ;
- .announced-checks = 1 ;
- }
-
- ECHO $(result) ;
- # FIXME: Unfinished code. Nothing seems to set .check-results at the moment.
- #.check-results += $(result) ;
-}
-
-
-rule log-library-search-result ( library : result )
-{
- local x = [ PAD " - $(library)" : $(.width) ] ;
- log-check-result "$(x) : $(result)" ;
-}
-
-
-rule print-component-configuration ( )
-{
- # FIXME: See what was intended with this initial assignment.
- # local c = [ sequence.unique $(.components) ] ;
-
- ECHO "\nComponent configuration:\n" ;
- local c ;
- for c in $(.components)
- {
- local s ;
- if $(c) in $(.built-components)
- {
- s = "building" ;
- }
- else
- {
- s = "not building" ;
- }
- ECHO [ PAD " - $(c)" : $(.width) ] ": $(s)" ;
- for local m in $(.component-logs.$(c))
- {
- ECHO " -" $(m) ;
- }
- }
- ECHO ;
-}
-
-
-rule print-configure-checks-summary ( )
-{
- # FIXME: The problem with this approach is that the user sees the checks
- # summary when all checks are done, and has no progress reporting while the
- # checks are being executed.
- if $(.check-results)
- {
- ECHO "Configuration checks summary\n" ;
- for local r in $(.check-results)
- {
- ECHO $(r) ;
- }
- ECHO ;
- }
-}
-
-
-# Attempt to build a metatarget named by 'metatarget-reference' in context of
-# 'project' with properties 'ps'. Returns non-empty value if build is OK.
-#
-rule builds-raw ( metatarget-reference : project : ps : what : retry ? )
-{
- local result ;
-
- if ! $(retry) && ! $(.$(what)-tested.$(ps))
- {
- .$(what)-tested.$(ps) = true ;
-
- local cache-name = $(what) [ $(ps).raw ] ;
- cache-name = $(cache-name:J=-) ;
- local value = [ config-cache.get $(cache-name) ] ;
-
- local targets = [ targets.generate-from-reference
- $(metatarget-reference) : $(project) : $(ps) ] ;
-
- local jam-targets ;
- for local t in $(targets[2-])
- {
- jam-targets += [ $(t).actualize ] ;
- }
-
- if $(value)
- {
- local x = [ PAD " - $(what)" : $(.width) ] ;
- if $(value) = true
- {
- .$(what)-supported.$(ps) = yes ;
- result = true ;
- log-check-result "$(x) : yes (cached)" ;
- }
- else
- {
- log-check-result "$(x) : no (cached)" ;
- }
- }
- else if ! UPDATE_NOW in [ RULENAMES ]
- {
- # Cannot determine. Assume existance.
- }
- else
- {
- local x = [ PAD " - $(what)" : $(.width) ] ;
- if [ UPDATE_NOW $(jam-targets) :
- $(.log-fd) : ignore-minus-n : ignore-minus-q ]
- {
- .$(what)-supported.$(ps) = yes ;
- result = true ;
- log-check-result "$(x) : yes" ;
- }
- else
- {
- log-check-result "$(x) : no" ;
- }
- }
- if ! $(value)
- {
- if $(result)
- {
- config-cache.set $(cache-name) : true ;
- }
- else
- {
- config-cache.set $(cache-name) : false ;
- }
- }
- return $(result) ;
- }
- else
- {
- return $(.$(what)-supported.$(ps)) ;
- }
-}
-
-rule builds ( metatarget-reference : properties * : what ? : retry ? )
-{
- # FIXME: This should not be hardcoded. Other checks might want to consider a
- # different set of features as relevant.
- local toolset = [ property.select <toolset> : $(properties) ] ;
- local toolset-version-property = "<toolset-$(toolset:G=):version>" ;
- local relevant = [ property.select <target-os> <toolset>
- $(toolset-version-property) <address-model> <architecture> :
- $(properties) ] ;
- local ps = [ property-set.create $(relevant) ] ;
- local t = [ targets.current ] ;
- local p = [ $(t).project ] ;
-
- if ! $(what)
- {
- local resolved = [ targets.resolve-reference $(metatarget-reference) : $(p) ] ;
- local name = [ $(resolved[1]).name ] ;
- what = "$(name) builds" ;
- }
-
- return [ builds-raw $(metatarget-reference) : $(p) : $(ps) : $(what) :
- $(retry) ] ;
-}
-
-
-# Called by Boost.Build startup code to specify the file to receive the
-# configuration check results. Should never be called by user code.
-#
-rule set-log-file ( log-file )
-{
- path.makedirs [ path.parent $(log-file) ] ;
- .log-fd = [ FILE_OPEN $(log-file) : "w" ] ;
-}
-
-
-# Frontend rules
-
-class check-target-builds-worker
-{
- import configure ;
- import property-set ;
- import targets ;
- import property ;
-
- rule __init__ ( target message ? : true-properties * : false-properties * )
- {
- self.target = $(target) ;
- self.message = $(message) ;
- self.true-properties = $(true-properties) ;
- self.false-properties = $(false-properties) ;
- }
-
- rule check ( properties * )
- {
- local choosen ;
- if [ configure.builds $(self.target) : $(properties) : $(self.message) ]
- {
- choosen = $(self.true-properties) ;
- }
- else
- {
- choosen = $(self.false-properties) ;
- }
- return [ property.evaluate-conditionals-in-context $(choosen) :
- $(properties) ] ;
- }
-}
-
-
-rule check-target-builds ( target message ? : true-properties * :
- false-properties * )
-{
- local instance = [ new check-target-builds-worker $(target) $(message) :
- $(true-properties) : $(false-properties) ] ;
- return <conditional>@$(instance).check ;
-}
-
-
-IMPORT $(__name__) : check-target-builds : : check-target-builds ;
[21/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/jamgram.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/jamgram.c b/ext/kenlm/jam-files/engine/jamgram.c
deleted file mode 100644
index 48c8522..0000000
--- a/ext/kenlm/jam-files/engine/jamgram.c
+++ /dev/null
@@ -1,2327 +0,0 @@
-/* A Bison parser, made by GNU Bison 2.4.3. */
-
-/* Skeleton implementation for Bison's Yacc-like parsers in C
-
- Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
- 2009, 2010 Free Software Foundation, Inc.
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see <http://www.gnu.org/licenses/>. */
-
-/* As a special exception, you may create a larger work that contains
- part or all of the Bison parser skeleton and distribute that work
- under terms of your choice, so long as that work isn't itself a
- parser generator using the skeleton or a modified version thereof
- as a parser skeleton. Alternatively, if you modify or redistribute
- the parser skeleton itself, you may (at your option) remove this
- special exception, which will cause the skeleton and the resulting
- Bison output files to be licensed under the GNU General Public
- License without this special exception.
-
- This special exception was added by the Free Software Foundation in
- version 2.2 of Bison. */
-
-/* C LALR(1) parser skeleton written by Richard Stallman, by
- simplifying the original so-called "semantic" parser. */
-
-/* All symbols defined below should begin with yy or YY, to avoid
- infringing on user name space. This should be done even for local
- variables, as they might otherwise be expanded by user macros.
- There are some unavoidable exceptions within include files to
- define necessary library symbols; they are noted "INFRINGES ON
- USER NAME SPACE" below. */
-
-/* Identify Bison output. */
-#define YYBISON 1
-
-/* Bison version. */
-#define YYBISON_VERSION "2.4.3"
-
-/* Skeleton name. */
-#define YYSKELETON_NAME "yacc.c"
-
-/* Pure parsers. */
-#define YYPURE 0
-
-/* Push parsers. */
-#define YYPUSH 0
-
-/* Pull parsers. */
-#define YYPULL 1
-
-/* Using locations. */
-#define YYLSP_NEEDED 0
-
-
-
-/* Copy the first part of user declarations. */
-
-/* Line 189 of yacc.c */
-#line 96 "jamgram.y"
-
-#include "jam.h"
-
-#include "lists.h"
-#include "parse.h"
-#include "scan.h"
-#include "compile.h"
-#include "object.h"
-#include "rules.h"
-
-# define YYMAXDEPTH 10000 /* for OSF and other less endowed yaccs */
-
-# define F0 -1
-# define P0 (PARSE *)0
-# define S0 (OBJECT *)0
-
-# define pappend( l,r ) parse_make( PARSE_APPEND,l,r,P0,S0,S0,0 )
-# define peval( c,l,r ) parse_make( PARSE_EVAL,l,r,P0,S0,S0,c )
-# define pfor( s,l,r,x ) parse_make( PARSE_FOREACH,l,r,P0,s,S0,x )
-# define pif( l,r,t ) parse_make( PARSE_IF,l,r,t,S0,S0,0 )
-# define pincl( l ) parse_make( PARSE_INCLUDE,l,P0,P0,S0,S0,0 )
-# define plist( s ) parse_make( PARSE_LIST,P0,P0,P0,s,S0,0 )
-# define plocal( l,r,t ) parse_make( PARSE_LOCAL,l,r,t,S0,S0,0 )
-# define pmodule( l,r ) parse_make( PARSE_MODULE,l,r,P0,S0,S0,0 )
-# define pclass( l,r ) parse_make( PARSE_CLASS,l,r,P0,S0,S0,0 )
-# define pnull() parse_make( PARSE_NULL,P0,P0,P0,S0,S0,0 )
-# define pon( l,r ) parse_make( PARSE_ON,l,r,P0,S0,S0,0 )
-# define prule( s,p ) parse_make( PARSE_RULE,p,P0,P0,s,S0,0 )
-# define prules( l,r ) parse_make( PARSE_RULES,l,r,P0,S0,S0,0 )
-# define pset( l,r,a ) parse_make( PARSE_SET,l,r,P0,S0,S0,a )
-# define pset1( l,r,t,a ) parse_make( PARSE_SETTINGS,l,r,t,S0,S0,a )
-# define psetc( s,p,a,l ) parse_make( PARSE_SETCOMP,p,a,P0,s,S0,l )
-# define psete( s,l,s1,f ) parse_make( PARSE_SETEXEC,l,P0,P0,s,s1,f )
-# define pswitch( l,r ) parse_make( PARSE_SWITCH,l,r,P0,S0,S0,0 )
-# define pwhile( l,r ) parse_make( PARSE_WHILE,l,r,P0,S0,S0,0 )
-
-# define pnode( l,r ) parse_make( F0,l,r,P0,S0,S0,0 )
-# define psnode( s,l ) parse_make( F0,l,P0,P0,s,S0,0 )
-
-
-
-/* Line 189 of yacc.c */
-#line 114 "y.tab.c"
-
-/* Enabling traces. */
-#ifndef YYDEBUG
-# define YYDEBUG 0
-#endif
-
-/* Enabling verbose error messages. */
-#ifdef YYERROR_VERBOSE
-# undef YYERROR_VERBOSE
-# define YYERROR_VERBOSE 1
-#else
-# define YYERROR_VERBOSE 0
-#endif
-
-/* Enabling the token table. */
-#ifndef YYTOKEN_TABLE
-# define YYTOKEN_TABLE 0
-#endif
-
-
-/* Tokens. */
-#ifndef YYTOKENTYPE
-# define YYTOKENTYPE
- /* Put the tokens into the symbol table, so that GDB and other debuggers
- know about them. */
- enum yytokentype {
- _BANG_t = 258,
- _BANG_EQUALS_t = 259,
- _AMPER_t = 260,
- _AMPERAMPER_t = 261,
- _LPAREN_t = 262,
- _RPAREN_t = 263,
- _PLUS_EQUALS_t = 264,
- _COLON_t = 265,
- _SEMIC_t = 266,
- _LANGLE_t = 267,
- _LANGLE_EQUALS_t = 268,
- _EQUALS_t = 269,
- _RANGLE_t = 270,
- _RANGLE_EQUALS_t = 271,
- _QUESTION_EQUALS_t = 272,
- _LBRACKET_t = 273,
- _RBRACKET_t = 274,
- ACTIONS_t = 275,
- BIND_t = 276,
- CASE_t = 277,
- CLASS_t = 278,
- DEFAULT_t = 279,
- ELSE_t = 280,
- EXISTING_t = 281,
- FOR_t = 282,
- IF_t = 283,
- IGNORE_t = 284,
- IN_t = 285,
- INCLUDE_t = 286,
- LOCAL_t = 287,
- MODULE_t = 288,
- ON_t = 289,
- PIECEMEAL_t = 290,
- QUIETLY_t = 291,
- RETURN_t = 292,
- RULE_t = 293,
- SWITCH_t = 294,
- TOGETHER_t = 295,
- UPDATED_t = 296,
- WHILE_t = 297,
- _LBRACE_t = 298,
- _BAR_t = 299,
- _BARBAR_t = 300,
- _RBRACE_t = 301,
- ARG = 302,
- STRING = 303
- };
-#endif
-/* Tokens. */
-#define _BANG_t 258
-#define _BANG_EQUALS_t 259
-#define _AMPER_t 260
-#define _AMPERAMPER_t 261
-#define _LPAREN_t 262
-#define _RPAREN_t 263
-#define _PLUS_EQUALS_t 264
-#define _COLON_t 265
-#define _SEMIC_t 266
-#define _LANGLE_t 267
-#define _LANGLE_EQUALS_t 268
-#define _EQUALS_t 269
-#define _RANGLE_t 270
-#define _RANGLE_EQUALS_t 271
-#define _QUESTION_EQUALS_t 272
-#define _LBRACKET_t 273
-#define _RBRACKET_t 274
-#define ACTIONS_t 275
-#define BIND_t 276
-#define CASE_t 277
-#define CLASS_t 278
-#define DEFAULT_t 279
-#define ELSE_t 280
-#define EXISTING_t 281
-#define FOR_t 282
-#define IF_t 283
-#define IGNORE_t 284
-#define IN_t 285
-#define INCLUDE_t 286
-#define LOCAL_t 287
-#define MODULE_t 288
-#define ON_t 289
-#define PIECEMEAL_t 290
-#define QUIETLY_t 291
-#define RETURN_t 292
-#define RULE_t 293
-#define SWITCH_t 294
-#define TOGETHER_t 295
-#define UPDATED_t 296
-#define WHILE_t 297
-#define _LBRACE_t 298
-#define _BAR_t 299
-#define _BARBAR_t 300
-#define _RBRACE_t 301
-#define ARG 302
-#define STRING 303
-
-
-
-
-#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED
-typedef int YYSTYPE;
-# define YYSTYPE_IS_TRIVIAL 1
-# define yystype YYSTYPE /* obsolescent; will be withdrawn */
-# define YYSTYPE_IS_DECLARED 1
-#endif
-
-
-/* Copy the second part of user declarations. */
-
-
-/* Line 264 of yacc.c */
-#line 252 "y.tab.c"
-
-#ifdef short
-# undef short
-#endif
-
-#ifdef YYTYPE_UINT8
-typedef YYTYPE_UINT8 yytype_uint8;
-#else
-typedef unsigned char yytype_uint8;
-#endif
-
-#ifdef YYTYPE_INT8
-typedef YYTYPE_INT8 yytype_int8;
-#elif (defined __STDC__ || defined __C99__FUNC__ \
- || defined __cplusplus || defined _MSC_VER)
-typedef signed char yytype_int8;
-#else
-typedef short int yytype_int8;
-#endif
-
-#ifdef YYTYPE_UINT16
-typedef YYTYPE_UINT16 yytype_uint16;
-#else
-typedef unsigned short int yytype_uint16;
-#endif
-
-#ifdef YYTYPE_INT16
-typedef YYTYPE_INT16 yytype_int16;
-#else
-typedef short int yytype_int16;
-#endif
-
-#ifndef YYSIZE_T
-# ifdef __SIZE_TYPE__
-# define YYSIZE_T __SIZE_TYPE__
-# elif defined size_t
-# define YYSIZE_T size_t
-# elif ! defined YYSIZE_T && (defined __STDC__ || defined __C99__FUNC__ \
- || defined __cplusplus || defined _MSC_VER)
-# include <stddef.h> /* INFRINGES ON USER NAME SPACE */
-# define YYSIZE_T size_t
-# else
-# define YYSIZE_T unsigned int
-# endif
-#endif
-
-#define YYSIZE_MAXIMUM ((YYSIZE_T) -1)
-
-#ifndef YY_
-# if defined YYENABLE_NLS && YYENABLE_NLS
-# if ENABLE_NLS
-# include <libintl.h> /* INFRINGES ON USER NAME SPACE */
-# define YY_(msgid) dgettext ("bison-runtime", msgid)
-# endif
-# endif
-# ifndef YY_
-# define YY_(msgid) msgid
-# endif
-#endif
-
-/* Suppress unused-variable warnings by "using" E. */
-#if ! defined lint || defined __GNUC__
-# define YYUSE(e) ((void) (e))
-#else
-# define YYUSE(e) /* empty */
-#endif
-
-/* Identity function, used to suppress warnings about constant conditions. */
-#ifndef lint
-# define YYID(n) (n)
-#else
-#if (defined __STDC__ || defined __C99__FUNC__ \
- || defined __cplusplus || defined _MSC_VER)
-static int
-YYID (int yyi)
-#else
-static int
-YYID (yyi)
- int yyi;
-#endif
-{
- return yyi;
-}
-#endif
-
-#if ! defined yyoverflow || YYERROR_VERBOSE
-
-/* The parser invokes alloca or malloc; define the necessary symbols. */
-
-# ifdef YYSTACK_USE_ALLOCA
-# if YYSTACK_USE_ALLOCA
-# ifdef __GNUC__
-# define YYSTACK_ALLOC __builtin_alloca
-# elif defined __BUILTIN_VA_ARG_INCR
-# include <alloca.h> /* INFRINGES ON USER NAME SPACE */
-# elif defined _AIX
-# define YYSTACK_ALLOC __alloca
-# elif defined _MSC_VER
-# include <malloc.h> /* INFRINGES ON USER NAME SPACE */
-# define alloca _alloca
-# else
-# define YYSTACK_ALLOC alloca
-# if ! defined _ALLOCA_H && ! defined _STDLIB_H && (defined __STDC__ || defined __C99__FUNC__ \
- || defined __cplusplus || defined _MSC_VER)
-# include <stdlib.h> /* INFRINGES ON USER NAME SPACE */
-# ifndef _STDLIB_H
-# define _STDLIB_H 1
-# endif
-# endif
-# endif
-# endif
-# endif
-
-# ifdef YYSTACK_ALLOC
- /* Pacify GCC's `empty if-body' warning. */
-# define YYSTACK_FREE(Ptr) do { /* empty */; } while (YYID (0))
-# ifndef YYSTACK_ALLOC_MAXIMUM
- /* The OS might guarantee only one guard page at the bottom of the stack,
- and a page size can be as small as 4096 bytes. So we cannot safely
- invoke alloca (N) if N exceeds 4096. Use a slightly smaller number
- to allow for a few compiler-allocated temporary stack slots. */
-# define YYSTACK_ALLOC_MAXIMUM 4032 /* reasonable circa 2006 */
-# endif
-# else
-# define YYSTACK_ALLOC YYMALLOC
-# define YYSTACK_FREE YYFREE
-# ifndef YYSTACK_ALLOC_MAXIMUM
-# define YYSTACK_ALLOC_MAXIMUM YYSIZE_MAXIMUM
-# endif
-# if (defined __cplusplus && ! defined _STDLIB_H \
- && ! ((defined YYMALLOC || defined malloc) \
- && (defined YYFREE || defined free)))
-# include <stdlib.h> /* INFRINGES ON USER NAME SPACE */
-# ifndef _STDLIB_H
-# define _STDLIB_H 1
-# endif
-# endif
-# ifndef YYMALLOC
-# define YYMALLOC malloc
-# if ! defined malloc && ! defined _STDLIB_H && (defined __STDC__ || defined __C99__FUNC__ \
- || defined __cplusplus || defined _MSC_VER)
-void *malloc (YYSIZE_T); /* INFRINGES ON USER NAME SPACE */
-# endif
-# endif
-# ifndef YYFREE
-# define YYFREE free
-# if ! defined free && ! defined _STDLIB_H && (defined __STDC__ || defined __C99__FUNC__ \
- || defined __cplusplus || defined _MSC_VER)
-void free (void *); /* INFRINGES ON USER NAME SPACE */
-# endif
-# endif
-# endif
-#endif /* ! defined yyoverflow || YYERROR_VERBOSE */
-
-
-#if (! defined yyoverflow \
- && (! defined __cplusplus \
- || (defined YYSTYPE_IS_TRIVIAL && YYSTYPE_IS_TRIVIAL)))
-
-/* A type that is properly aligned for any stack member. */
-union yyalloc
-{
- yytype_int16 yyss_alloc;
- YYSTYPE yyvs_alloc;
-};
-
-/* The size of the maximum gap between one aligned stack and the next. */
-# define YYSTACK_GAP_MAXIMUM (sizeof (union yyalloc) - 1)
-
-/* The size of an array large to enough to hold all stacks, each with
- N elements. */
-# define YYSTACK_BYTES(N) \
- ((N) * (sizeof (yytype_int16) + sizeof (YYSTYPE)) \
- + YYSTACK_GAP_MAXIMUM)
-
-/* Copy COUNT objects from FROM to TO. The source and destination do
- not overlap. */
-# ifndef YYCOPY
-# if defined __GNUC__ && 1 < __GNUC__
-# define YYCOPY(To, From, Count) \
- __builtin_memcpy (To, From, (Count) * sizeof (*(From)))
-# else
-# define YYCOPY(To, From, Count) \
- do \
- { \
- YYSIZE_T yyi; \
- for (yyi = 0; yyi < (Count); yyi++) \
- (To)[yyi] = (From)[yyi]; \
- } \
- while (YYID (0))
-# endif
-# endif
-
-/* Relocate STACK from its old location to the new one. The
- local variables YYSIZE and YYSTACKSIZE give the old and new number of
- elements in the stack, and YYPTR gives the new location of the
- stack. Advance YYPTR to a properly aligned location for the next
- stack. */
-# define YYSTACK_RELOCATE(Stack_alloc, Stack) \
- do \
- { \
- YYSIZE_T yynewbytes; \
- YYCOPY (&yyptr->Stack_alloc, Stack, yysize); \
- Stack = &yyptr->Stack_alloc; \
- yynewbytes = yystacksize * sizeof (*Stack) + YYSTACK_GAP_MAXIMUM; \
- yyptr += yynewbytes / sizeof (*yyptr); \
- } \
- while (YYID (0))
-
-#endif
-
-/* YYFINAL -- State number of the termination state. */
-#define YYFINAL 43
-/* YYLAST -- Last index in YYTABLE. */
-#define YYLAST 243
-
-/* YYNTOKENS -- Number of terminals. */
-#define YYNTOKENS 49
-/* YYNNTS -- Number of nonterminals. */
-#define YYNNTS 24
-/* YYNRULES -- Number of rules. */
-#define YYNRULES 75
-/* YYNRULES -- Number of states. */
-#define YYNSTATES 159
-
-/* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX. */
-#define YYUNDEFTOK 2
-#define YYMAXUTOK 303
-
-#define YYTRANSLATE(YYX) \
- ((unsigned int) (YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK)
-
-/* YYTRANSLATE[YYLEX] -- Bison symbol number corresponding to YYLEX. */
-static const yytype_uint8 yytranslate[] =
-{
- 0, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
- 2, 2, 2, 2, 2, 2, 1, 2, 3, 4,
- 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
- 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,
- 25, 26, 27, 28, 29, 30, 31, 32, 33, 34,
- 35, 36, 37, 38, 39, 40, 41, 42, 43, 44,
- 45, 46, 47, 48
-};
-
-#if YYDEBUG
-/* YYPRHS[YYN] -- Index of the first RHS symbol of rule number YYN in
- YYRHS. */
-static const yytype_uint8 yyprhs[] =
-{
- 0, 0, 3, 4, 6, 8, 10, 12, 15, 21,
- 22, 25, 27, 31, 32, 34, 35, 39, 43, 47,
- 52, 59, 63, 72, 78, 84, 90, 96, 102, 110,
- 116, 120, 121, 122, 132, 134, 136, 138, 141, 143,
- 147, 151, 155, 159, 163, 167, 171, 175, 179, 183,
- 187, 190, 194, 195, 198, 203, 205, 209, 211, 212,
- 215, 217, 218, 223, 226, 231, 236, 237, 240, 242,
- 244, 246, 248, 250, 252, 253
-};
-
-/* YYRHS -- A `-1'-separated list of the rules' RHS. */
-static const yytype_int8 yyrhs[] =
-{
- 50, 0, -1, -1, 52, -1, 53, -1, 52, -1,
- 57, -1, 57, 52, -1, 32, 65, 54, 11, 51,
- -1, -1, 14, 65, -1, 53, -1, 7, 64, 8,
- -1, -1, 32, -1, -1, 43, 51, 46, -1, 31,
- 65, 11, -1, 47, 64, 11, -1, 67, 60, 65,
- 11, -1, 67, 34, 65, 60, 65, 11, -1, 37,
- 65, 11, -1, 27, 56, 47, 30, 65, 43, 51,
- 46, -1, 39, 65, 43, 62, 46, -1, 28, 61,
- 43, 51, 46, -1, 33, 65, 43, 51, 46, -1,
- 23, 64, 43, 51, 46, -1, 42, 61, 43, 51,
- 46, -1, 28, 61, 43, 51, 46, 25, 57, -1,
- 56, 38, 47, 55, 57, -1, 34, 67, 57, -1,
- -1, -1, 20, 70, 47, 72, 43, 58, 48, 59,
- 46, -1, 14, -1, 9, -1, 17, -1, 24, 14,
- -1, 67, -1, 61, 14, 61, -1, 61, 4, 61,
- -1, 61, 12, 61, -1, 61, 13, 61, -1, 61,
- 15, 61, -1, 61, 16, 61, -1, 61, 5, 61,
- -1, 61, 6, 61, -1, 61, 44, 61, -1, 61,
- 45, 61, -1, 67, 30, 65, -1, 3, 61, -1,
- 7, 61, 8, -1, -1, 63, 62, -1, 22, 47,
- 10, 51, -1, 65, -1, 65, 10, 64, -1, 66,
- -1, -1, 66, 67, -1, 47, -1, -1, 18, 68,
- 69, 19, -1, 47, 64, -1, 34, 67, 47, 64,
- -1, 34, 67, 37, 65, -1, -1, 70, 71, -1,
- 41, -1, 40, -1, 29, -1, 36, -1, 35, -1,
- 26, -1, -1, 21, 65, -1
-};
-
-/* YYRLINE[YYN] -- source line where rule number YYN was defined. */
-static const yytype_uint16 yyrline[] =
-{
- 0, 139, 139, 141, 152, 154, 158, 160, 162, 167,
- 170, 172, 176, 179, 182, 185, 188, 190, 192, 194,
- 196, 198, 200, 202, 204, 206, 208, 210, 212, 214,
- 216, 219, 221, 218, 230, 232, 234, 236, 243, 245,
- 247, 249, 251, 253, 255, 257, 259, 261, 263, 265,
- 267, 269, 281, 282, 286, 295, 297, 307, 312, 313,
- 317, 319, 319, 328, 330, 332, 343, 344, 348, 350,
- 352, 354, 356, 358, 368, 369
-};
-#endif
-
-#if YYDEBUG || YYERROR_VERBOSE || YYTOKEN_TABLE
-/* YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM.
- First, the terminals, then, starting at YYNTOKENS, nonterminals. */
-static const char *const yytname[] =
-{
- "$end", "error", "$undefined", "_BANG_t", "_BANG_EQUALS_t", "_AMPER_t",
- "_AMPERAMPER_t", "_LPAREN_t", "_RPAREN_t", "_PLUS_EQUALS_t", "_COLON_t",
- "_SEMIC_t", "_LANGLE_t", "_LANGLE_EQUALS_t", "_EQUALS_t", "_RANGLE_t",
- "_RANGLE_EQUALS_t", "_QUESTION_EQUALS_t", "_LBRACKET_t", "_RBRACKET_t",
- "ACTIONS_t", "BIND_t", "CASE_t", "CLASS_t", "DEFAULT_t", "ELSE_t",
- "EXISTING_t", "FOR_t", "IF_t", "IGNORE_t", "IN_t", "INCLUDE_t",
- "LOCAL_t", "MODULE_t", "ON_t", "PIECEMEAL_t", "QUIETLY_t", "RETURN_t",
- "RULE_t", "SWITCH_t", "TOGETHER_t", "UPDATED_t", "WHILE_t", "_LBRACE_t",
- "_BAR_t", "_BARBAR_t", "_RBRACE_t", "ARG", "STRING", "$accept", "run",
- "block", "rules", "null", "assign_list_opt", "arglist_opt", "local_opt",
- "rule", "$@1", "$@2", "assign", "expr", "cases", "case", "lol", "list",
- "listp", "arg", "$@3", "func", "eflags", "eflag", "bindlist", 0
-};
-#endif
-
-# ifdef YYPRINT
-/* YYTOKNUM[YYLEX-NUM] -- Internal token number corresponding to
- token YYLEX-NUM. */
-static const yytype_uint16 yytoknum[] =
-{
- 0, 256, 257, 258, 259, 260, 261, 262, 263, 264,
- 265, 266, 267, 268, 269, 270, 271, 272, 273, 274,
- 275, 276, 277, 278, 279, 280, 281, 282, 283, 284,
- 285, 286, 287, 288, 289, 290, 291, 292, 293, 294,
- 295, 296, 297, 298, 299, 300, 301, 302, 303
-};
-# endif
-
-/* YYR1[YYN] -- Symbol number of symbol that rule YYN derives. */
-static const yytype_uint8 yyr1[] =
-{
- 0, 49, 50, 50, 51, 51, 52, 52, 52, 53,
- 54, 54, 55, 55, 56, 56, 57, 57, 57, 57,
- 57, 57, 57, 57, 57, 57, 57, 57, 57, 57,
- 57, 58, 59, 57, 60, 60, 60, 60, 61, 61,
- 61, 61, 61, 61, 61, 61, 61, 61, 61, 61,
- 61, 61, 62, 62, 63, 64, 64, 65, 66, 66,
- 67, 68, 67, 69, 69, 69, 70, 70, 71, 71,
- 71, 71, 71, 71, 72, 72
-};
-
-/* YYR2[YYN] -- Number of symbols composing right hand side of rule YYN. */
-static const yytype_uint8 yyr2[] =
-{
- 0, 2, 0, 1, 1, 1, 1, 2, 5, 0,
- 2, 1, 3, 0, 1, 0, 3, 3, 3, 4,
- 6, 3, 8, 5, 5, 5, 5, 5, 7, 5,
- 3, 0, 0, 9, 1, 1, 1, 2, 1, 3,
- 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
- 2, 3, 0, 2, 4, 1, 3, 1, 0, 2,
- 1, 0, 4, 2, 4, 4, 0, 2, 1, 1,
- 1, 1, 1, 1, 0, 2
-};
-
-/* YYDEFACT[STATE-NAME] -- Default rule to reduce with in state
- STATE-NUM when YYTABLE doesn't specify something else to do. Zero
- means the default is an error. */
-static const yytype_uint8 yydefact[] =
-{
- 2, 61, 66, 58, 15, 0, 58, 58, 58, 0,
- 58, 58, 0, 9, 60, 0, 3, 0, 6, 0,
- 0, 0, 0, 55, 57, 14, 0, 0, 0, 60,
- 0, 38, 0, 9, 0, 15, 0, 0, 0, 0,
- 5, 4, 0, 1, 0, 7, 35, 34, 36, 0,
- 58, 58, 0, 58, 0, 73, 70, 72, 71, 69,
- 68, 74, 67, 9, 58, 59, 0, 50, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 9, 0, 0,
- 58, 17, 58, 11, 0, 9, 30, 21, 52, 9,
- 16, 18, 13, 37, 0, 0, 0, 63, 62, 58,
- 0, 0, 56, 58, 51, 40, 45, 46, 41, 42,
- 39, 43, 44, 0, 47, 48, 49, 10, 9, 0,
- 0, 0, 52, 0, 58, 15, 58, 19, 58, 58,
- 75, 31, 26, 0, 24, 8, 25, 0, 23, 53,
- 27, 0, 29, 0, 65, 64, 0, 9, 15, 9,
- 12, 20, 32, 0, 28, 54, 0, 22, 33
-};
-
-/* YYDEFGOTO[NTERM-NUM]. */
-static const yytype_int16 yydefgoto[] =
-{
- -1, 15, 39, 40, 41, 84, 125, 17, 18, 146,
- 156, 51, 30, 121, 122, 22, 23, 24, 31, 20,
- 54, 21, 62, 100
-};
-
-/* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing
- STATE-NUM. */
-#define YYPACT_NINF -48
-static const yytype_int16 yypact[] =
-{
- 170, -48, -48, -48, -12, 7, -48, -17, -48, -3,
- -48, -48, 7, 170, 1, 22, -48, -9, 170, 19,
- -2, 79, -6, 29, -3, -48, 2, 7, 7, -48,
- 138, 20, 44, 45, 18, 196, 51, 26, 151, 24,
- -48, -48, 62, -48, 27, -48, -48, -48, -48, 61,
- -48, -48, -3, -48, 67, -48, -48, -48, -48, -48,
- -48, 58, -48, 170, -48, -48, 50, -48, 52, 7,
- 7, 7, 7, 7, 7, 7, 7, 170, 7, 7,
- -48, -48, -48, -48, 70, 170, -48, -48, 87, 170,
- -48, -48, 94, -48, 17, 99, -20, -48, -48, -48,
- 69, 71, -48, -48, -48, 91, 156, 156, -48, -48,
- 91, -48, -48, 77, 78, 78, -48, -48, 170, 81,
- 66, 82, 87, 95, -48, 196, -48, -48, -48, -48,
- -48, -48, -48, 97, 112, -48, -48, 135, -48, -48,
- -48, 150, -48, 148, -48, -48, 98, 170, 196, 170,
- -48, -48, -48, 115, -48, -48, 116, -48, -48
-};
-
-/* YYPGOTO[NTERM-NUM]. */
-static const yytype_int16 yypgoto[] =
-{
- -48, -48, -47, 5, 140, -48, -48, 171, -27, -48,
- -48, 80, 60, 54, -48, -13, -4, -48, 0, -48,
- -48, -48, -48, -48
-};
-
-/* YYTABLE[YYPACT[STATE-NUM]]. What to do in state STATE-NUM. If
- positive, shift that token. If negative, reduce the rule which
- number is the opposite. If zero, do what YYDEFACT says.
- If YYTABLE_NINF, syntax error. */
-#define YYTABLE_NINF -59
-static const yytype_int16 yytable[] =
-{
- 19, 42, 32, 33, 34, 16, 36, 37, 86, 35,
- 27, -58, -58, 19, 28, 1, 101, 128, 19, -58,
- 25, -14, 43, 45, 65, 1, 46, 129, 46, 44,
- 113, 47, 52, 47, 48, 19, 48, 63, 119, 64,
- 97, 49, 123, 49, 29, 53, 94, 95, -58, 66,
- 80, 102, 96, 50, 29, 81, 69, 70, 71, 82,
- 104, 85, 87, 19, 72, 73, 74, 75, 76, 88,
- 90, 135, 38, 91, 92, 93, 116, 19, 117, 99,
- 103, 118, 69, 70, 71, 19, 98, 67, 68, 19,
- 72, 73, 74, 75, 76, 130, 78, 79, 142, 133,
- 153, 124, 155, 72, 73, 55, 75, 76, 56, 120,
- 127, 141, 131, 137, 57, 58, 145, 132, 19, 59,
- 60, 154, 143, 134, 144, 19, 61, 136, 138, 105,
- 106, 107, 108, 109, 110, 111, 112, 148, 114, 115,
- 147, 140, 69, 70, 71, 149, 152, 19, 19, 19,
- 72, 73, 74, 75, 76, 69, 70, 71, 150, 151,
- 69, 157, 158, 72, 73, 74, 75, 76, 72, 73,
- 74, 75, 76, 83, 126, 26, 139, 0, 0, 0,
- 0, 77, 78, 79, 0, 0, 0, 0, 1, 0,
- 2, 0, 0, 3, 89, 78, 79, 4, 5, 0,
- 0, 6, 7, 8, 9, 0, 0, 10, -15, 11,
- 0, 0, 12, 13, 1, 0, 2, 14, 0, 3,
- 0, 0, 0, 4, 5, 0, 0, 6, 25, 8,
- 9, 0, 0, 10, 0, 11, 0, 0, 12, 13,
- 0, 0, 0, 14
-};
-
-static const yytype_int16 yycheck[] =
-{
- 0, 14, 6, 7, 8, 0, 10, 11, 35, 9,
- 3, 10, 11, 13, 7, 18, 63, 37, 18, 18,
- 32, 38, 0, 18, 24, 18, 9, 47, 9, 38,
- 77, 14, 34, 14, 17, 35, 17, 43, 85, 10,
- 53, 24, 89, 24, 47, 47, 50, 51, 47, 47,
- 30, 64, 52, 34, 47, 11, 4, 5, 6, 14,
- 8, 43, 11, 63, 12, 13, 14, 15, 16, 43,
- 46, 118, 12, 11, 47, 14, 80, 77, 82, 21,
- 30, 11, 4, 5, 6, 85, 19, 27, 28, 89,
- 12, 13, 14, 15, 16, 99, 44, 45, 125, 103,
- 147, 7, 149, 12, 13, 26, 15, 16, 29, 22,
- 11, 124, 43, 47, 35, 36, 129, 46, 118, 40,
- 41, 148, 126, 46, 128, 125, 47, 46, 46, 69,
- 70, 71, 72, 73, 74, 75, 76, 25, 78, 79,
- 43, 46, 4, 5, 6, 10, 48, 147, 148, 149,
- 12, 13, 14, 15, 16, 4, 5, 6, 8, 11,
- 4, 46, 46, 12, 13, 14, 15, 16, 12, 13,
- 14, 15, 16, 33, 94, 4, 122, -1, -1, -1,
- -1, 43, 44, 45, -1, -1, -1, -1, 18, -1,
- 20, -1, -1, 23, 43, 44, 45, 27, 28, -1,
- -1, 31, 32, 33, 34, -1, -1, 37, 38, 39,
- -1, -1, 42, 43, 18, -1, 20, 47, -1, 23,
- -1, -1, -1, 27, 28, -1, -1, 31, 32, 33,
- 34, -1, -1, 37, -1, 39, -1, -1, 42, 43,
- -1, -1, -1, 47
-};
-
-/* YYSTOS[STATE-NUM] -- The (internal number of the) accessing
- symbol of state STATE-NUM. */
-static const yytype_uint8 yystos[] =
-{
- 0, 18, 20, 23, 27, 28, 31, 32, 33, 34,
- 37, 39, 42, 43, 47, 50, 52, 56, 57, 67,
- 68, 70, 64, 65, 66, 32, 56, 3, 7, 47,
- 61, 67, 65, 65, 65, 67, 65, 65, 61, 51,
- 52, 53, 64, 0, 38, 52, 9, 14, 17, 24,
- 34, 60, 34, 47, 69, 26, 29, 35, 36, 40,
- 41, 47, 71, 43, 10, 67, 47, 61, 61, 4,
- 5, 6, 12, 13, 14, 15, 16, 43, 44, 45,
- 30, 11, 14, 53, 54, 43, 57, 11, 43, 43,
- 46, 11, 47, 14, 65, 65, 67, 64, 19, 21,
- 72, 51, 64, 30, 8, 61, 61, 61, 61, 61,
- 61, 61, 61, 51, 61, 61, 65, 65, 11, 51,
- 22, 62, 63, 51, 7, 55, 60, 11, 37, 47,
- 65, 43, 46, 65, 46, 51, 46, 47, 46, 62,
- 46, 64, 57, 65, 65, 64, 58, 43, 25, 10,
- 8, 11, 48, 51, 57, 51, 59, 46, 46
-};
-
-#define yyerrok (yyerrstatus = 0)
-#define yyclearin (yychar = YYEMPTY)
-#define YYEMPTY (-2)
-#define YYEOF 0
-
-#define YYACCEPT goto yyacceptlab
-#define YYABORT goto yyabortlab
-#define YYERROR goto yyerrorlab
-
-
-/* Like YYERROR except do call yyerror. This remains here temporarily
- to ease the transition to the new meaning of YYERROR, for GCC.
- Once GCC version 2 has supplanted version 1, this can go. However,
- YYFAIL appears to be in use. Nevertheless, it is formally deprecated
- in Bison 2.4.2's NEWS entry, where a plan to phase it out is
- discussed. */
-
-#define YYFAIL goto yyerrlab
-#if defined YYFAIL
- /* This is here to suppress warnings from the GCC cpp's
- -Wunused-macros. Normally we don't worry about that warning, but
- some users do, and we want to make it easy for users to remove
- YYFAIL uses, which will produce warnings from Bison 2.5. */
-#endif
-
-#define YYRECOVERING() (!!yyerrstatus)
-
-#define YYBACKUP(Token, Value) \
-do \
- if (yychar == YYEMPTY && yylen == 1) \
- { \
- yychar = (Token); \
- yylval = (Value); \
- yytoken = YYTRANSLATE (yychar); \
- YYPOPSTACK (1); \
- goto yybackup; \
- } \
- else \
- { \
- yyerror (YY_("syntax error: cannot back up")); \
- YYERROR; \
- } \
-while (YYID (0))
-
-
-#define YYTERROR 1
-#define YYERRCODE 256
-
-
-/* YYLLOC_DEFAULT -- Set CURRENT to span from RHS[1] to RHS[N].
- If N is 0, then set CURRENT to the empty location which ends
- the previous symbol: RHS[0] (always defined). */
-
-#define YYRHSLOC(Rhs, K) ((Rhs)[K])
-#ifndef YYLLOC_DEFAULT
-# define YYLLOC_DEFAULT(Current, Rhs, N) \
- do \
- if (YYID (N)) \
- { \
- (Current).first_line = YYRHSLOC (Rhs, 1).first_line; \
- (Current).first_column = YYRHSLOC (Rhs, 1).first_column; \
- (Current).last_line = YYRHSLOC (Rhs, N).last_line; \
- (Current).last_column = YYRHSLOC (Rhs, N).last_column; \
- } \
- else \
- { \
- (Current).first_line = (Current).last_line = \
- YYRHSLOC (Rhs, 0).last_line; \
- (Current).first_column = (Current).last_column = \
- YYRHSLOC (Rhs, 0).last_column; \
- } \
- while (YYID (0))
-#endif
-
-
-/* YY_LOCATION_PRINT -- Print the location on the stream.
- This macro was not mandated originally: define only if we know
- we won't break user code: when these are the locations we know. */
-
-#ifndef YY_LOCATION_PRINT
-# if defined YYLTYPE_IS_TRIVIAL && YYLTYPE_IS_TRIVIAL
-# define YY_LOCATION_PRINT(File, Loc) \
- fprintf (File, "%d.%d-%d.%d", \
- (Loc).first_line, (Loc).first_column, \
- (Loc).last_line, (Loc).last_column)
-# else
-# define YY_LOCATION_PRINT(File, Loc) ((void) 0)
-# endif
-#endif
-
-
-/* YYLEX -- calling `yylex' with the right arguments. */
-
-#ifdef YYLEX_PARAM
-# define YYLEX yylex (YYLEX_PARAM)
-#else
-# define YYLEX yylex ()
-#endif
-
-/* Enable debugging if requested. */
-#if YYDEBUG
-
-# ifndef YYFPRINTF
-# include <stdio.h> /* INFRINGES ON USER NAME SPACE */
-# define YYFPRINTF fprintf
-# endif
-
-# define YYDPRINTF(Args) \
-do { \
- if (yydebug) \
- YYFPRINTF Args; \
-} while (YYID (0))
-
-# define YY_SYMBOL_PRINT(Title, Type, Value, Location) \
-do { \
- if (yydebug) \
- { \
- YYFPRINTF (stderr, "%s ", Title); \
- yy_symbol_print (stderr, \
- Type, Value); \
- YYFPRINTF (stderr, "\n"); \
- } \
-} while (YYID (0))
-
-
-/*--------------------------------.
-| Print this symbol on YYOUTPUT. |
-`--------------------------------*/
-
-/*ARGSUSED*/
-#if (defined __STDC__ || defined __C99__FUNC__ \
- || defined __cplusplus || defined _MSC_VER)
-static void
-yy_symbol_value_print (FILE *yyoutput, int yytype, YYSTYPE const * const yyvaluep)
-#else
-static void
-yy_symbol_value_print (yyoutput, yytype, yyvaluep)
- FILE *yyoutput;
- int yytype;
- YYSTYPE const * const yyvaluep;
-#endif
-{
- if (!yyvaluep)
- return;
-# ifdef YYPRINT
- if (yytype < YYNTOKENS)
- YYPRINT (yyoutput, yytoknum[yytype], *yyvaluep);
-# else
- YYUSE (yyoutput);
-# endif
- switch (yytype)
- {
- default:
- break;
- }
-}
-
-
-/*--------------------------------.
-| Print this symbol on YYOUTPUT. |
-`--------------------------------*/
-
-#if (defined __STDC__ || defined __C99__FUNC__ \
- || defined __cplusplus || defined _MSC_VER)
-static void
-yy_symbol_print (FILE *yyoutput, int yytype, YYSTYPE const * const yyvaluep)
-#else
-static void
-yy_symbol_print (yyoutput, yytype, yyvaluep)
- FILE *yyoutput;
- int yytype;
- YYSTYPE const * const yyvaluep;
-#endif
-{
- if (yytype < YYNTOKENS)
- YYFPRINTF (yyoutput, "token %s (", yytname[yytype]);
- else
- YYFPRINTF (yyoutput, "nterm %s (", yytname[yytype]);
-
- yy_symbol_value_print (yyoutput, yytype, yyvaluep);
- YYFPRINTF (yyoutput, ")");
-}
-
-/*------------------------------------------------------------------.
-| yy_stack_print -- Print the state stack from its BOTTOM up to its |
-| TOP (included). |
-`------------------------------------------------------------------*/
-
-#if (defined __STDC__ || defined __C99__FUNC__ \
- || defined __cplusplus || defined _MSC_VER)
-static void
-yy_stack_print (yytype_int16 *yybottom, yytype_int16 *yytop)
-#else
-static void
-yy_stack_print (yybottom, yytop)
- yytype_int16 *yybottom;
- yytype_int16 *yytop;
-#endif
-{
- YYFPRINTF (stderr, "Stack now");
- for (; yybottom <= yytop; yybottom++)
- {
- int yybot = *yybottom;
- YYFPRINTF (stderr, " %d", yybot);
- }
- YYFPRINTF (stderr, "\n");
-}
-
-# define YY_STACK_PRINT(Bottom, Top) \
-do { \
- if (yydebug) \
- yy_stack_print ((Bottom), (Top)); \
-} while (YYID (0))
-
-
-/*------------------------------------------------.
-| Report that the YYRULE is going to be reduced. |
-`------------------------------------------------*/
-
-#if (defined __STDC__ || defined __C99__FUNC__ \
- || defined __cplusplus || defined _MSC_VER)
-static void
-yy_reduce_print (YYSTYPE *yyvsp, int yyrule)
-#else
-static void
-yy_reduce_print (yyvsp, yyrule)
- YYSTYPE *yyvsp;
- int yyrule;
-#endif
-{
- int yynrhs = yyr2[yyrule];
- int yyi;
- unsigned long int yylno = yyrline[yyrule];
- YYFPRINTF (stderr, "Reducing stack by rule %d (line %lu):\n",
- yyrule - 1, yylno);
- /* The symbols being reduced. */
- for (yyi = 0; yyi < yynrhs; yyi++)
- {
- YYFPRINTF (stderr, " $%d = ", yyi + 1);
- yy_symbol_print (stderr, yyrhs[yyprhs[yyrule] + yyi],
- &(yyvsp[(yyi + 1) - (yynrhs)])
- );
- YYFPRINTF (stderr, "\n");
- }
-}
-
-# define YY_REDUCE_PRINT(Rule) \
-do { \
- if (yydebug) \
- yy_reduce_print (yyvsp, Rule); \
-} while (YYID (0))
-
-/* Nonzero means print parse trace. It is left uninitialized so that
- multiple parsers can coexist. */
-int yydebug;
-#else /* !YYDEBUG */
-# define YYDPRINTF(Args)
-# define YY_SYMBOL_PRINT(Title, Type, Value, Location)
-# define YY_STACK_PRINT(Bottom, Top)
-# define YY_REDUCE_PRINT(Rule)
-#endif /* !YYDEBUG */
-
-
-/* YYINITDEPTH -- initial size of the parser's stacks. */
-#ifndef YYINITDEPTH
-# define YYINITDEPTH 200
-#endif
-
-/* YYMAXDEPTH -- maximum size the stacks can grow to (effective only
- if the built-in stack extension method is used).
-
- Do not make this value too large; the results are undefined if
- YYSTACK_ALLOC_MAXIMUM < YYSTACK_BYTES (YYMAXDEPTH)
- evaluated with infinite-precision integer arithmetic. */
-
-#ifndef YYMAXDEPTH
-# define YYMAXDEPTH 10000
-#endif
-
-
-
-#if YYERROR_VERBOSE
-
-# ifndef yystrlen
-# if defined __GLIBC__ && defined _STRING_H
-# define yystrlen strlen
-# else
-/* Return the length of YYSTR. */
-#if (defined __STDC__ || defined __C99__FUNC__ \
- || defined __cplusplus || defined _MSC_VER)
-static YYSIZE_T
-yystrlen (const char *yystr)
-#else
-static YYSIZE_T
-yystrlen (yystr)
- const char *yystr;
-#endif
-{
- YYSIZE_T yylen;
- for (yylen = 0; yystr[yylen]; yylen++)
- continue;
- return yylen;
-}
-# endif
-# endif
-
-# ifndef yystpcpy
-# if defined __GLIBC__ && defined _STRING_H && defined _GNU_SOURCE
-# define yystpcpy stpcpy
-# else
-/* Copy YYSRC to YYDEST, returning the address of the terminating '\0' in
- YYDEST. */
-#if (defined __STDC__ || defined __C99__FUNC__ \
- || defined __cplusplus || defined _MSC_VER)
-static char *
-yystpcpy (char *yydest, const char *yysrc)
-#else
-static char *
-yystpcpy (yydest, yysrc)
- char *yydest;
- const char *yysrc;
-#endif
-{
- char *yyd = yydest;
- const char *yys = yysrc;
-
- while ((*yyd++ = *yys++) != '\0')
- continue;
-
- return yyd - 1;
-}
-# endif
-# endif
-
-# ifndef yytnamerr
-/* Copy to YYRES the contents of YYSTR after stripping away unnecessary
- quotes and backslashes, so that it's suitable for yyerror. The
- heuristic is that double-quoting is unnecessary unless the string
- contains an apostrophe, a comma, or backslash (other than
- backslash-backslash). YYSTR is taken from yytname. If YYRES is
- null, do not copy; instead, return the length of what the result
- would have been. */
-static YYSIZE_T
-yytnamerr (char *yyres, const char *yystr)
-{
- if (*yystr == '"')
- {
- YYSIZE_T yyn = 0;
- char const *yyp = yystr;
-
- for (;;)
- switch (*++yyp)
- {
- case '\'':
- case ',':
- goto do_not_strip_quotes;
-
- case '\\':
- if (*++yyp != '\\')
- goto do_not_strip_quotes;
- /* Fall through. */
- default:
- if (yyres)
- yyres[yyn] = *yyp;
- yyn++;
- break;
-
- case '"':
- if (yyres)
- yyres[yyn] = '\0';
- return yyn;
- }
- do_not_strip_quotes: ;
- }
-
- if (! yyres)
- return yystrlen (yystr);
-
- return yystpcpy (yyres, yystr) - yyres;
-}
-# endif
-
-/* Copy into YYRESULT an error message about the unexpected token
- YYCHAR while in state YYSTATE. Return the number of bytes copied,
- including the terminating null byte. If YYRESULT is null, do not
- copy anything; just return the number of bytes that would be
- copied. As a special case, return 0 if an ordinary "syntax error"
- message will do. Return YYSIZE_MAXIMUM if overflow occurs during
- size calculation. */
-static YYSIZE_T
-yysyntax_error (char *yyresult, int yystate, int yychar)
-{
- int yyn = yypact[yystate];
-
- if (! (YYPACT_NINF < yyn && yyn <= YYLAST))
- return 0;
- else
- {
- int yytype = YYTRANSLATE (yychar);
- YYSIZE_T yysize0 = yytnamerr (0, yytname[yytype]);
- YYSIZE_T yysize = yysize0;
- YYSIZE_T yysize1;
- int yysize_overflow = 0;
- enum { YYERROR_VERBOSE_ARGS_MAXIMUM = 5 };
- char const *yyarg[YYERROR_VERBOSE_ARGS_MAXIMUM];
- int yyx;
-
-# if 0
- /* This is so xgettext sees the translatable formats that are
- constructed on the fly. */
- YY_("syntax error, unexpected %s");
- YY_("syntax error, unexpected %s, expecting %s");
- YY_("syntax error, unexpected %s, expecting %s or %s");
- YY_("syntax error, unexpected %s, expecting %s or %s or %s");
- YY_("syntax error, unexpected %s, expecting %s or %s or %s or %s");
-# endif
- char *yyfmt;
- char const *yyf;
- static char const yyunexpected[] = "syntax error, unexpected %s";
- static char const yyexpecting[] = ", expecting %s";
- static char const yyor[] = " or %s";
- char yyformat[sizeof yyunexpected
- + sizeof yyexpecting - 1
- + ((YYERROR_VERBOSE_ARGS_MAXIMUM - 2)
- * (sizeof yyor - 1))];
- char const *yyprefix = yyexpecting;
-
- /* Start YYX at -YYN if negative to avoid negative indexes in
- YYCHECK. */
- int yyxbegin = yyn < 0 ? -yyn : 0;
-
- /* Stay within bounds of both yycheck and yytname. */
- int yychecklim = YYLAST - yyn + 1;
- int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS;
- int yycount = 1;
-
- yyarg[0] = yytname[yytype];
- yyfmt = yystpcpy (yyformat, yyunexpected);
-
- for (yyx = yyxbegin; yyx < yyxend; ++yyx)
- if (yycheck[yyx + yyn] == yyx && yyx != YYTERROR)
- {
- if (yycount == YYERROR_VERBOSE_ARGS_MAXIMUM)
- {
- yycount = 1;
- yysize = yysize0;
- yyformat[sizeof yyunexpected - 1] = '\0';
- break;
- }
- yyarg[yycount++] = yytname[yyx];
- yysize1 = yysize + yytnamerr (0, yytname[yyx]);
- yysize_overflow |= (yysize1 < yysize);
- yysize = yysize1;
- yyfmt = yystpcpy (yyfmt, yyprefix);
- yyprefix = yyor;
- }
-
- yyf = YY_(yyformat);
- yysize1 = yysize + yystrlen (yyf);
- yysize_overflow |= (yysize1 < yysize);
- yysize = yysize1;
-
- if (yysize_overflow)
- return YYSIZE_MAXIMUM;
-
- if (yyresult)
- {
- /* Avoid sprintf, as that infringes on the user's name space.
- Don't have undefined behavior even if the translation
- produced a string with the wrong number of "%s"s. */
- char *yyp = yyresult;
- int yyi = 0;
- while ((*yyp = *yyf) != '\0')
- {
- if (*yyp == '%' && yyf[1] == 's' && yyi < yycount)
- {
- yyp += yytnamerr (yyp, yyarg[yyi++]);
- yyf += 2;
- }
- else
- {
- yyp++;
- yyf++;
- }
- }
- }
- return yysize;
- }
-}
-#endif /* YYERROR_VERBOSE */
-
-
-/*-----------------------------------------------.
-| Release the memory associated to this symbol. |
-`-----------------------------------------------*/
-
-/*ARGSUSED*/
-#if (defined __STDC__ || defined __C99__FUNC__ \
- || defined __cplusplus || defined _MSC_VER)
-static void
-yydestruct (const char *yymsg, int yytype, YYSTYPE *yyvaluep)
-#else
-static void
-yydestruct (yymsg, yytype, yyvaluep)
- const char *yymsg;
- int yytype;
- YYSTYPE *yyvaluep;
-#endif
-{
- YYUSE (yyvaluep);
-
- if (!yymsg)
- yymsg = "Deleting";
- YY_SYMBOL_PRINT (yymsg, yytype, yyvaluep, yylocationp);
-
- switch (yytype)
- {
-
- default:
- break;
- }
-}
-
-/* Prevent warnings from -Wmissing-prototypes. */
-#ifdef YYPARSE_PARAM
-#if defined __STDC__ || defined __cplusplus
-int yyparse (void *YYPARSE_PARAM);
-#else
-int yyparse ();
-#endif
-#else /* ! YYPARSE_PARAM */
-#if defined __STDC__ || defined __cplusplus
-int yyparse (void);
-#else
-int yyparse ();
-#endif
-#endif /* ! YYPARSE_PARAM */
-
-
-/* The lookahead symbol. */
-int yychar;
-
-/* The semantic value of the lookahead symbol. */
-YYSTYPE yylval;
-
-/* Number of syntax errors so far. */
-int yynerrs;
-
-
-
-/*-------------------------.
-| yyparse or yypush_parse. |
-`-------------------------*/
-
-#ifdef YYPARSE_PARAM
-#if (defined __STDC__ || defined __C99__FUNC__ \
- || defined __cplusplus || defined _MSC_VER)
-int
-yyparse (void *YYPARSE_PARAM)
-#else
-int
-yyparse (YYPARSE_PARAM)
- void *YYPARSE_PARAM;
-#endif
-#else /* ! YYPARSE_PARAM */
-#if (defined __STDC__ || defined __C99__FUNC__ \
- || defined __cplusplus || defined _MSC_VER)
-int
-yyparse (void)
-#else
-int
-yyparse ()
-
-#endif
-#endif
-{
-
-
- int yystate;
- /* Number of tokens to shift before error messages enabled. */
- int yyerrstatus;
-
- /* The stacks and their tools:
- `yyss': related to states.
- `yyvs': related to semantic values.
-
- Refer to the stacks thru separate pointers, to allow yyoverflow
- to reallocate them elsewhere. */
-
- /* The state stack. */
- yytype_int16 yyssa[YYINITDEPTH];
- yytype_int16 *yyss;
- yytype_int16 *yyssp;
-
- /* The semantic value stack. */
- YYSTYPE yyvsa[YYINITDEPTH];
- YYSTYPE *yyvs;
- YYSTYPE *yyvsp;
-
- YYSIZE_T yystacksize;
-
- int yyn;
- int yyresult;
- /* Lookahead token as an internal (translated) token number. */
- int yytoken;
- /* The variables used to return semantic value and location from the
- action routines. */
- YYSTYPE yyval;
-
-#if YYERROR_VERBOSE
- /* Buffer for error messages, and its allocated size. */
- char yymsgbuf[128];
- char *yymsg = yymsgbuf;
- YYSIZE_T yymsg_alloc = sizeof yymsgbuf;
-#endif
-
-#define YYPOPSTACK(N) (yyvsp -= (N), yyssp -= (N))
-
- /* The number of symbols on the RHS of the reduced rule.
- Keep to zero when no symbol should be popped. */
- int yylen = 0;
-
- yytoken = 0;
- yyss = yyssa;
- yyvs = yyvsa;
- yystacksize = YYINITDEPTH;
-
- YYDPRINTF ((stderr, "Starting parse\n"));
-
- yystate = 0;
- yyerrstatus = 0;
- yynerrs = 0;
- yychar = YYEMPTY; /* Cause a token to be read. */
-
- /* Initialize stack pointers.
- Waste one element of value and location stack
- so that they stay on the same level as the state stack.
- The wasted elements are never initialized. */
- yyssp = yyss;
- yyvsp = yyvs;
-
- goto yysetstate;
-
-/*------------------------------------------------------------.
-| yynewstate -- Push a new state, which is found in yystate. |
-`------------------------------------------------------------*/
- yynewstate:
- /* In all cases, when you get here, the value and location stacks
- have just been pushed. So pushing a state here evens the stacks. */
- yyssp++;
-
- yysetstate:
- *yyssp = yystate;
-
- if (yyss + yystacksize - 1 <= yyssp)
- {
- /* Get the current used size of the three stacks, in elements. */
- YYSIZE_T yysize = yyssp - yyss + 1;
-
-#ifdef yyoverflow
- {
- /* Give user a chance to reallocate the stack. Use copies of
- these so that the &'s don't force the real ones into
- memory. */
- YYSTYPE *yyvs1 = yyvs;
- yytype_int16 *yyss1 = yyss;
-
- /* Each stack pointer address is followed by the size of the
- data in use in that stack, in bytes. This used to be a
- conditional around just the two extra args, but that might
- be undefined if yyoverflow is a macro. */
- yyoverflow (YY_("memory exhausted"),
- &yyss1, yysize * sizeof (*yyssp),
- &yyvs1, yysize * sizeof (*yyvsp),
- &yystacksize);
-
- yyss = yyss1;
- yyvs = yyvs1;
- }
-#else /* no yyoverflow */
-# ifndef YYSTACK_RELOCATE
- goto yyexhaustedlab;
-# else
- /* Extend the stack our own way. */
- if (YYMAXDEPTH <= yystacksize)
- goto yyexhaustedlab;
- yystacksize *= 2;
- if (YYMAXDEPTH < yystacksize)
- yystacksize = YYMAXDEPTH;
-
- {
- yytype_int16 *yyss1 = yyss;
- union yyalloc *yyptr =
- (union yyalloc *) YYSTACK_ALLOC (YYSTACK_BYTES (yystacksize));
- if (! yyptr)
- goto yyexhaustedlab;
- YYSTACK_RELOCATE (yyss_alloc, yyss);
- YYSTACK_RELOCATE (yyvs_alloc, yyvs);
-# undef YYSTACK_RELOCATE
- if (yyss1 != yyssa)
- YYSTACK_FREE (yyss1);
- }
-# endif
-#endif /* no yyoverflow */
-
- yyssp = yyss + yysize - 1;
- yyvsp = yyvs + yysize - 1;
-
- YYDPRINTF ((stderr, "Stack size increased to %lu\n",
- (unsigned long int) yystacksize));
-
- if (yyss + yystacksize - 1 <= yyssp)
- YYABORT;
- }
-
- YYDPRINTF ((stderr, "Entering state %d\n", yystate));
-
- if (yystate == YYFINAL)
- YYACCEPT;
-
- goto yybackup;
-
-/*-----------.
-| yybackup. |
-`-----------*/
-yybackup:
-
- /* Do appropriate processing given the current state. Read a
- lookahead token if we need one and don't already have one. */
-
- /* First try to decide what to do without reference to lookahead token. */
- yyn = yypact[yystate];
- if (yyn == YYPACT_NINF)
- goto yydefault;
-
- /* Not known => get a lookahead token if don't already have one. */
-
- /* YYCHAR is either YYEMPTY or YYEOF or a valid lookahead symbol. */
- if (yychar == YYEMPTY)
- {
- YYDPRINTF ((stderr, "Reading a token: "));
- yychar = YYLEX;
- }
-
- if (yychar <= YYEOF)
- {
- yychar = yytoken = YYEOF;
- YYDPRINTF ((stderr, "Now at end of input.\n"));
- }
- else
- {
- yytoken = YYTRANSLATE (yychar);
- YY_SYMBOL_PRINT ("Next token is", yytoken, &yylval, &yylloc);
- }
-
- /* If the proper action on seeing token YYTOKEN is to reduce or to
- detect an error, take that action. */
- yyn += yytoken;
- if (yyn < 0 || YYLAST < yyn || yycheck[yyn] != yytoken)
- goto yydefault;
- yyn = yytable[yyn];
- if (yyn <= 0)
- {
- if (yyn == 0 || yyn == YYTABLE_NINF)
- goto yyerrlab;
- yyn = -yyn;
- goto yyreduce;
- }
-
- /* Count tokens shifted since error; after three, turn off error
- status. */
- if (yyerrstatus)
- yyerrstatus--;
-
- /* Shift the lookahead token. */
- YY_SYMBOL_PRINT ("Shifting", yytoken, &yylval, &yylloc);
-
- /* Discard the shifted token. */
- yychar = YYEMPTY;
-
- yystate = yyn;
- *++yyvsp = yylval;
-
- goto yynewstate;
-
-
-/*-----------------------------------------------------------.
-| yydefault -- do the default action for the current state. |
-`-----------------------------------------------------------*/
-yydefault:
- yyn = yydefact[yystate];
- if (yyn == 0)
- goto yyerrlab;
- goto yyreduce;
-
-
-/*-----------------------------.
-| yyreduce -- Do a reduction. |
-`-----------------------------*/
-yyreduce:
- /* yyn is the number of a rule to reduce with. */
- yylen = yyr2[yyn];
-
- /* If YYLEN is nonzero, implement the default value of the action:
- `$$ = $1'.
-
- Otherwise, the following line sets YYVAL to garbage.
- This behavior is undocumented and Bison
- users should not rely upon it. Assigning to YYVAL
- unconditionally makes the parser a bit smaller, and it avoids a
- GCC warning that YYVAL may be used uninitialized. */
- yyval = yyvsp[1-yylen];
-
-
- YY_REDUCE_PRINT (yyn);
- switch (yyn)
- {
- case 3:
-
-/* Line 1464 of yacc.c */
-#line 142 "jamgram.y"
- { parse_save( (yyvsp[(1) - (1)]).parse ); }
- break;
-
- case 4:
-
-/* Line 1464 of yacc.c */
-#line 153 "jamgram.y"
- { (yyval).parse = (yyvsp[(1) - (1)]).parse; }
- break;
-
- case 5:
-
-/* Line 1464 of yacc.c */
-#line 155 "jamgram.y"
- { (yyval).parse = (yyvsp[(1) - (1)]).parse; }
- break;
-
- case 6:
-
-/* Line 1464 of yacc.c */
-#line 159 "jamgram.y"
- { (yyval).parse = (yyvsp[(1) - (1)]).parse; }
- break;
-
- case 7:
-
-/* Line 1464 of yacc.c */
-#line 161 "jamgram.y"
- { (yyval).parse = prules( (yyvsp[(1) - (2)]).parse, (yyvsp[(2) - (2)]).parse ); }
- break;
-
- case 8:
-
-/* Line 1464 of yacc.c */
-#line 163 "jamgram.y"
- { (yyval).parse = plocal( (yyvsp[(2) - (5)]).parse, (yyvsp[(3) - (5)]).parse, (yyvsp[(5) - (5)]).parse ); }
- break;
-
- case 9:
-
-/* Line 1464 of yacc.c */
-#line 167 "jamgram.y"
- { (yyval).parse = pnull(); }
- break;
-
- case 10:
-
-/* Line 1464 of yacc.c */
-#line 171 "jamgram.y"
- { (yyval).parse = (yyvsp[(2) - (2)]).parse; (yyval).number = ASSIGN_SET; }
- break;
-
- case 11:
-
-/* Line 1464 of yacc.c */
-#line 173 "jamgram.y"
- { (yyval).parse = (yyvsp[(1) - (1)]).parse; (yyval).number = ASSIGN_APPEND; }
- break;
-
- case 12:
-
-/* Line 1464 of yacc.c */
-#line 177 "jamgram.y"
- { (yyval).parse = (yyvsp[(2) - (3)]).parse; }
- break;
-
- case 13:
-
-/* Line 1464 of yacc.c */
-#line 179 "jamgram.y"
- { (yyval).parse = P0; }
- break;
-
- case 14:
-
-/* Line 1464 of yacc.c */
-#line 183 "jamgram.y"
- { (yyval).number = 1; }
- break;
-
- case 15:
-
-/* Line 1464 of yacc.c */
-#line 185 "jamgram.y"
- { (yyval).number = 0; }
- break;
-
- case 16:
-
-/* Line 1464 of yacc.c */
-#line 189 "jamgram.y"
- { (yyval).parse = (yyvsp[(2) - (3)]).parse; }
- break;
-
- case 17:
-
-/* Line 1464 of yacc.c */
-#line 191 "jamgram.y"
- { (yyval).parse = pincl( (yyvsp[(2) - (3)]).parse ); }
- break;
-
- case 18:
-
-/* Line 1464 of yacc.c */
-#line 193 "jamgram.y"
- { (yyval).parse = prule( (yyvsp[(1) - (3)]).string, (yyvsp[(2) - (3)]).parse ); }
- break;
-
- case 19:
-
-/* Line 1464 of yacc.c */
-#line 195 "jamgram.y"
- { (yyval).parse = pset( (yyvsp[(1) - (4)]).parse, (yyvsp[(3) - (4)]).parse, (yyvsp[(2) - (4)]).number ); }
- break;
-
- case 20:
-
-/* Line 1464 of yacc.c */
-#line 197 "jamgram.y"
- { (yyval).parse = pset1( (yyvsp[(1) - (6)]).parse, (yyvsp[(3) - (6)]).parse, (yyvsp[(5) - (6)]).parse, (yyvsp[(4) - (6)]).number ); }
- break;
-
- case 21:
-
-/* Line 1464 of yacc.c */
-#line 199 "jamgram.y"
- { (yyval).parse = (yyvsp[(2) - (3)]).parse; }
- break;
-
- case 22:
-
-/* Line 1464 of yacc.c */
-#line 201 "jamgram.y"
- { (yyval).parse = pfor( (yyvsp[(3) - (8)]).string, (yyvsp[(5) - (8)]).parse, (yyvsp[(7) - (8)]).parse, (yyvsp[(2) - (8)]).number ); }
- break;
-
- case 23:
-
-/* Line 1464 of yacc.c */
-#line 203 "jamgram.y"
- { (yyval).parse = pswitch( (yyvsp[(2) - (5)]).parse, (yyvsp[(4) - (5)]).parse ); }
- break;
-
- case 24:
-
-/* Line 1464 of yacc.c */
-#line 205 "jamgram.y"
- { (yyval).parse = pif( (yyvsp[(2) - (5)]).parse, (yyvsp[(4) - (5)]).parse, pnull() ); }
- break;
-
- case 25:
-
-/* Line 1464 of yacc.c */
-#line 207 "jamgram.y"
- { (yyval).parse = pmodule( (yyvsp[(2) - (5)]).parse, (yyvsp[(4) - (5)]).parse ); }
- break;
-
- case 26:
-
-/* Line 1464 of yacc.c */
-#line 209 "jamgram.y"
- { (yyval).parse = pclass( (yyvsp[(2) - (5)]).parse, (yyvsp[(4) - (5)]).parse ); }
- break;
-
- case 27:
-
-/* Line 1464 of yacc.c */
-#line 211 "jamgram.y"
- { (yyval).parse = pwhile( (yyvsp[(2) - (5)]).parse, (yyvsp[(4) - (5)]).parse ); }
- break;
-
- case 28:
-
-/* Line 1464 of yacc.c */
-#line 213 "jamgram.y"
- { (yyval).parse = pif( (yyvsp[(2) - (7)]).parse, (yyvsp[(4) - (7)]).parse, (yyvsp[(7) - (7)]).parse ); }
- break;
-
- case 29:
-
-/* Line 1464 of yacc.c */
-#line 215 "jamgram.y"
- { (yyval).parse = psetc( (yyvsp[(3) - (5)]).string, (yyvsp[(5) - (5)]).parse, (yyvsp[(4) - (5)]).parse, (yyvsp[(1) - (5)]).number ); }
- break;
-
- case 30:
-
-/* Line 1464 of yacc.c */
-#line 217 "jamgram.y"
- { (yyval).parse = pon( (yyvsp[(2) - (3)]).parse, (yyvsp[(3) - (3)]).parse ); }
- break;
-
- case 31:
-
-/* Line 1464 of yacc.c */
-#line 219 "jamgram.y"
- { yymode( SCAN_STRING ); }
- break;
-
- case 32:
-
-/* Line 1464 of yacc.c */
-#line 221 "jamgram.y"
- { yymode( SCAN_NORMAL ); }
- break;
-
- case 33:
-
-/* Line 1464 of yacc.c */
-#line 223 "jamgram.y"
- { (yyval).parse = psete( (yyvsp[(3) - (9)]).string,(yyvsp[(4) - (9)]).parse,(yyvsp[(7) - (9)]).string,(yyvsp[(2) - (9)]).number ); }
- break;
-
- case 34:
-
-/* Line 1464 of yacc.c */
-#line 231 "jamgram.y"
- { (yyval).number = ASSIGN_SET; }
- break;
-
- case 35:
-
-/* Line 1464 of yacc.c */
-#line 233 "jamgram.y"
- { (yyval).number = ASSIGN_APPEND; }
- break;
-
- case 36:
-
-/* Line 1464 of yacc.c */
-#line 235 "jamgram.y"
- { (yyval).number = ASSIGN_DEFAULT; }
- break;
-
- case 37:
-
-/* Line 1464 of yacc.c */
-#line 237 "jamgram.y"
- { (yyval).number = ASSIGN_DEFAULT; }
- break;
-
- case 38:
-
-/* Line 1464 of yacc.c */
-#line 244 "jamgram.y"
- { (yyval).parse = peval( EXPR_EXISTS, (yyvsp[(1) - (1)]).parse, pnull() ); }
- break;
-
- case 39:
-
-/* Line 1464 of yacc.c */
-#line 246 "jamgram.y"
- { (yyval).parse = peval( EXPR_EQUALS, (yyvsp[(1) - (3)]).parse, (yyvsp[(3) - (3)]).parse ); }
- break;
-
- case 40:
-
-/* Line 1464 of yacc.c */
-#line 248 "jamgram.y"
- { (yyval).parse = peval( EXPR_NOTEQ, (yyvsp[(1) - (3)]).parse, (yyvsp[(3) - (3)]).parse ); }
- break;
-
- case 41:
-
-/* Line 1464 of yacc.c */
-#line 250 "jamgram.y"
- { (yyval).parse = peval( EXPR_LESS, (yyvsp[(1) - (3)]).parse, (yyvsp[(3) - (3)]).parse ); }
- break;
-
- case 42:
-
-/* Line 1464 of yacc.c */
-#line 252 "jamgram.y"
- { (yyval).parse = peval( EXPR_LESSEQ, (yyvsp[(1) - (3)]).parse, (yyvsp[(3) - (3)]).parse ); }
- break;
-
- case 43:
-
-/* Line 1464 of yacc.c */
-#line 254 "jamgram.y"
- { (yyval).parse = peval( EXPR_MORE, (yyvsp[(1) - (3)]).parse, (yyvsp[(3) - (3)]).parse ); }
- break;
-
- case 44:
-
-/* Line 1464 of yacc.c */
-#line 256 "jamgram.y"
- { (yyval).parse = peval( EXPR_MOREEQ, (yyvsp[(1) - (3)]).parse, (yyvsp[(3) - (3)]).parse ); }
- break;
-
- case 45:
-
-/* Line 1464 of yacc.c */
-#line 258 "jamgram.y"
- { (yyval).parse = peval( EXPR_AND, (yyvsp[(1) - (3)]).parse, (yyvsp[(3) - (3)]).parse ); }
- break;
-
- case 46:
-
-/* Line 1464 of yacc.c */
-#line 260 "jamgram.y"
- { (yyval).parse = peval( EXPR_AND, (yyvsp[(1) - (3)]).parse, (yyvsp[(3) - (3)]).parse ); }
- break;
-
- case 47:
-
-/* Line 1464 of yacc.c */
-#line 262 "jamgram.y"
- { (yyval).parse = peval( EXPR_OR, (yyvsp[(1) - (3)]).parse, (yyvsp[(3) - (3)]).parse ); }
- break;
-
- case 48:
-
-/* Line 1464 of yacc.c */
-#line 264 "jamgram.y"
- { (yyval).parse = peval( EXPR_OR, (yyvsp[(1) - (3)]).parse, (yyvsp[(3) - (3)]).parse ); }
- break;
-
- case 49:
-
-/* Line 1464 of yacc.c */
-#line 266 "jamgram.y"
- { (yyval).parse = peval( EXPR_IN, (yyvsp[(1) - (3)]).parse, (yyvsp[(3) - (3)]).parse ); }
- break;
-
- case 50:
-
-/* Line 1464 of yacc.c */
-#line 268 "jamgram.y"
- { (yyval).parse = peval( EXPR_NOT, (yyvsp[(2) - (2)]).parse, pnull() ); }
- break;
-
- case 51:
-
-/* Line 1464 of yacc.c */
-#line 270 "jamgram.y"
- { (yyval).parse = (yyvsp[(2) - (3)]).parse; }
- break;
-
- case 52:
-
-/* Line 1464 of yacc.c */
-#line 281 "jamgram.y"
- { (yyval).parse = P0; }
- break;
-
- case 53:
-
-/* Line 1464 of yacc.c */
-#line 283 "jamgram.y"
- { (yyval).parse = pnode( (yyvsp[(1) - (2)]).parse, (yyvsp[(2) - (2)]).parse ); }
- break;
-
- case 54:
-
-/* Line 1464 of yacc.c */
-#line 287 "jamgram.y"
- { (yyval).parse = psnode( (yyvsp[(2) - (4)]).string, (yyvsp[(4) - (4)]).parse ); }
- break;
-
- case 55:
-
-/* Line 1464 of yacc.c */
-#line 296 "jamgram.y"
- { (yyval).parse = pnode( P0, (yyvsp[(1) - (1)]).parse ); }
- break;
-
- case 56:
-
-/* Line 1464 of yacc.c */
-#line 298 "jamgram.y"
- { (yyval).parse = pnode( (yyvsp[(3) - (3)]).parse, (yyvsp[(1) - (3)]).parse ); }
- break;
-
- case 57:
-
-/* Line 1464 of yacc.c */
-#line 308 "jamgram.y"
- { (yyval).parse = (yyvsp[(1) - (1)]).parse; yymode( SCAN_NORMAL ); }
- break;
-
- case 58:
-
-/* Line 1464 of yacc.c */
-#line 312 "jamgram.y"
- { (yyval).parse = pnull(); yymode( SCAN_PUNCT ); }
- break;
-
- case 59:
-
-/* Line 1464 of yacc.c */
-#line 314 "jamgram.y"
- { (yyval).parse = pappend( (yyvsp[(1) - (2)]).parse, (yyvsp[(2) - (2)]).parse ); }
- break;
-
- case 60:
-
-/* Line 1464 of yacc.c */
-#line 318 "jamgram.y"
- { (yyval).parse = plist( (yyvsp[(1) - (1)]).string ); }
- break;
-
- case 61:
-
-/* Line 1464 of yacc.c */
-#line 319 "jamgram.y"
- { yymode( SCAN_NORMAL ); }
- break;
-
- case 62:
-
-/* Line 1464 of yacc.c */
-#line 320 "jamgram.y"
- { (yyval).parse = (yyvsp[(3) - (4)]).parse; }
- break;
-
- case 63:
-
-/* Line 1464 of yacc.c */
-#line 329 "jamgram.y"
- { (yyval).parse = prule( (yyvsp[(1) - (2)]).string, (yyvsp[(2) - (2)]).parse ); }
- break;
-
- case 64:
-
-/* Line 1464 of yacc.c */
-#line 331 "jamgram.y"
- { (yyval).parse = pon( (yyvsp[(2) - (4)]).parse, prule( (yyvsp[(3) - (4)]).string, (yyvsp[(4) - (4)]).parse ) ); }
- break;
-
- case 65:
-
-/* Line 1464 of yacc.c */
-#line 333 "jamgram.y"
- { (yyval).parse = pon( (yyvsp[(2) - (4)]).parse, (yyvsp[(4) - (4)]).parse ); }
- break;
-
- case 66:
-
-/* Line 1464 of yacc.c */
-#line 343 "jamgram.y"
- { (yyval).number = 0; }
- break;
-
- case 67:
-
-/* Line 1464 of yacc.c */
-#line 345 "jamgram.y"
- { (yyval).number = (yyvsp[(1) - (2)]).number | (yyvsp[(2) - (2)]).number; }
- break;
-
- case 68:
-
-/* Line 1464 of yacc.c */
-#line 349 "jamgram.y"
- { (yyval).number = EXEC_UPDATED; }
- break;
-
- case 69:
-
-/* Line 1464 of yacc.c */
-#line 351 "jamgram.y"
- { (yyval).number = EXEC_TOGETHER; }
- break;
-
- case 70:
-
-/* Line 1464 of yacc.c */
-#line 353 "jamgram.y"
- { (yyval).number = EXEC_IGNORE; }
- break;
-
- case 71:
-
-/* Line 1464 of yacc.c */
-#line 355 "jamgram.y"
- { (yyval).number = EXEC_QUIETLY; }
- break;
-
- case 72:
-
-/* Line 1464 of yacc.c */
-#line 357 "jamgram.y"
- { (yyval).number = EXEC_PIECEMEAL; }
- break;
-
- case 73:
-
-/* Line 1464 of yacc.c */
-#line 359 "jamgram.y"
- { (yyval).number = EXEC_EXISTING; }
- break;
-
- case 74:
-
-/* Line 1464 of yacc.c */
-#line 368 "jamgram.y"
- { (yyval).parse = pnull(); }
- break;
-
- case 75:
-
-/* Line 1464 of yacc.c */
-#line 370 "jamgram.y"
- { (yyval).parse = (yyvsp[(2) - (2)]).parse; }
- break;
-
-
-
-/* Line 1464 of yacc.c */
-#line 2118 "y.tab.c"
- default: break;
- }
- YY_SYMBOL_PRINT ("-> $$ =", yyr1[yyn], &yyval, &yyloc);
-
- YYPOPSTACK (yylen);
- yylen = 0;
- YY_STACK_PRINT (yyss, yyssp);
-
- *++yyvsp = yyval;
-
- /* Now `shift' the result of the reduction. Determine what state
- that goes to, based on the state we popped back to and the rule
- number reduced by. */
-
- yyn = yyr1[yyn];
-
- yystate = yypgoto[yyn - YYNTOKENS] + *yyssp;
- if (0 <= yystate && yystate <= YYLAST && yycheck[yystate] == *yyssp)
- yystate = yytable[yystate];
- else
- yystate = yydefgoto[yyn - YYNTOKENS];
-
- goto yynewstate;
-
-
-/*------------------------------------.
-| yyerrlab -- here on detecting error |
-`------------------------------------*/
-yyerrlab:
- /* If not already recovering from an error, report this error. */
- if (!yyerrstatus)
- {
- ++yynerrs;
-#if ! YYERROR_VERBOSE
- yyerror (YY_("syntax error"));
-#else
- {
- YYSIZE_T yysize = yysyntax_error (0, yystate, yychar);
- if (yymsg_alloc < yysize && yymsg_alloc < YYSTACK_ALLOC_MAXIMUM)
- {
- YYSIZE_T yyalloc = 2 * yysize;
- if (! (yysize <= yyalloc && yyalloc <= YYSTACK_ALLOC_MAXIMUM))
- yyalloc = YYSTACK_ALLOC_MAXIMUM;
- if (yymsg != yymsgbuf)
- YYSTACK_FREE (yymsg);
- yymsg = (char *) YYSTACK_ALLOC (yyalloc);
- if (yymsg)
- yymsg_alloc = yyalloc;
- else
- {
- yymsg = yymsgbuf;
- yymsg_alloc = sizeof yymsgbuf;
- }
- }
-
- if (0 < yysize && yysize <= yymsg_alloc)
- {
- (void) yysyntax_error (yymsg, yystate, yychar);
- yyerror (yymsg);
- }
- else
- {
- yyerror (YY_("syntax error"));
- if (yysize != 0)
- goto yyexhaustedlab;
- }
- }
-#endif
- }
-
-
-
- if (yyerrstatus == 3)
- {
- /* If just tried and failed to reuse lookahead token after an
- error, discard it. */
-
- if (yychar <= YYEOF)
- {
- /* Return failure if at end of input. */
- if (yychar == YYEOF)
- YYABORT;
- }
- else
- {
- yydestruct ("Error: discarding",
- yytoken, &yylval);
- yychar = YYEMPTY;
- }
- }
-
- /* Else will try to reuse lookahead token after shifting the error
- token. */
- goto yyerrlab1;
-
-
-/*---------------------------------------------------.
-| yyerrorlab -- error raised explicitly by YYERROR. |
-`---------------------------------------------------*/
-yyerrorlab:
-
- /* Pacify compilers like GCC when the user code never invokes
- YYERROR and the label yyerrorlab therefore never appears in user
- code. */
- if (/*CONSTCOND*/ 0)
- goto yyerrorlab;
-
- /* Do not reclaim the symbols of the rule which action triggered
- this YYERROR. */
- YYPOPSTACK (yylen);
- yylen = 0;
- YY_STACK_PRINT (yyss, yyssp);
- yystate = *yyssp;
- goto yyerrlab1;
-
-
-/*-------------------------------------------------------------.
-| yyerrlab1 -- common code for both syntax error and YYERROR. |
-`-------------------------------------------------------------*/
-yyerrlab1:
- yyerrstatus = 3; /* Each real token shifted decrements this. */
-
- for (;;)
- {
- yyn = yypact[yystate];
- if (yyn != YYPACT_NINF)
- {
- yyn += YYTERROR;
- if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == YYTERROR)
- {
- yyn = yytable[yyn];
- if (0 < yyn)
- break;
- }
- }
-
- /* Pop the current state because it cannot handle the error token. */
- if (yyssp == yyss)
- YYABORT;
-
-
- yydestruct ("Error: popping",
- yystos[yystate], yyvsp);
- YYPOPSTACK (1);
- yystate = *yyssp;
- YY_STACK_PRINT (yyss, yyssp);
- }
-
- *++yyvsp = yylval;
-
-
- /* Shift the error token. */
- YY_SYMBOL_PRINT ("Shifting", yystos[yyn], yyvsp, yylsp);
-
- yystate = yyn;
- goto yynewstate;
-
-
-/*-------------------------------------.
-| yyacceptlab -- YYACCEPT comes here. |
-`-------------------------------------*/
-yyacceptlab:
- yyresult = 0;
- goto yyreturn;
-
-/*-----------------------------------.
-| yyabortlab -- YYABORT comes here. |
-`-----------------------------------*/
-yyabortlab:
- yyresult = 1;
- goto yyreturn;
-
-#if !defined(yyoverflow) || YYERROR_VERBOSE
-/*-------------------------------------------------.
-| yyexhaustedlab -- memory exhaustion comes here. |
-`-------------------------------------------------*/
-yyexhaustedlab:
- yyerror (YY_("memory exhausted"));
- yyresult = 2;
- /* Fall through. */
-#endif
-
-yyreturn:
- if (yychar != YYEMPTY)
- yydestruct ("Cleanup: discarding lookahead",
- yytoken, &yylval);
- /* Do not reclaim the symbols of the rule which action triggered
- this YYABORT or YYACCEPT. */
- YYPOPSTACK (yylen);
- YY_STACK_PRINT (yyss, yyssp);
- while (yyssp != yyss)
- {
- yydestruct ("Cleanup: popping",
- yystos[*yyssp], yyvsp);
- YYPOPSTACK (1);
- }
-#ifndef yyoverflow
- if (yyss != yyssa)
- YYSTACK_FREE (yyss);
-#endif
-#if YYERROR_VERBOSE
- if (yymsg != yymsgbuf)
- YYSTACK_FREE (yymsg);
-#endif
- /* Make sure YYID is used. */
- return YYID (yyresult);
-}
-
-
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/jamgram.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/jamgram.h b/ext/kenlm/jam-files/engine/jamgram.h
deleted file mode 100644
index 97f1175..0000000
--- a/ext/kenlm/jam-files/engine/jamgram.h
+++ /dev/null
@@ -1,149 +0,0 @@
-/* A Bison parser, made by GNU Bison 2.4.3. */
-
-/* Skeleton interface for Bison's Yacc-like parsers in C
-
- Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
- 2009, 2010 Free Software Foundation, Inc.
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see <http://www.gnu.org/licenses/>. */
-
-/* As a special exception, you may create a larger work that contains
- part or all of the Bison parser skeleton and distribute that work
- under terms of your choice, so long as that work isn't itself a
- parser generator using the skeleton or a modified version thereof
- as a parser skeleton. Alternatively, if you modify or redistribute
- the parser skeleton itself, you may (at your option) remove this
- special exception, which will cause the skeleton and the resulting
- Bison output files to be licensed under the GNU General Public
- License without this special exception.
-
- This special exception was added by the Free Software Foundation in
- version 2.2 of Bison. */
-
-
-/* Tokens. */
-#ifndef YYTOKENTYPE
-# define YYTOKENTYPE
- /* Put the tokens into the symbol table, so that GDB and other debuggers
- know about them. */
- enum yytokentype {
- _BANG_t = 258,
- _BANG_EQUALS_t = 259,
- _AMPER_t = 260,
- _AMPERAMPER_t = 261,
- _LPAREN_t = 262,
- _RPAREN_t = 263,
- _PLUS_EQUALS_t = 264,
- _COLON_t = 265,
- _SEMIC_t = 266,
- _LANGLE_t = 267,
- _LANGLE_EQUALS_t = 268,
- _EQUALS_t = 269,
- _RANGLE_t = 270,
- _RANGLE_EQUALS_t = 271,
- _QUESTION_EQUALS_t = 272,
- _LBRACKET_t = 273,
- _RBRACKET_t = 274,
- ACTIONS_t = 275,
- BIND_t = 276,
- CASE_t = 277,
- CLASS_t = 278,
- DEFAULT_t = 279,
- ELSE_t = 280,
- EXISTING_t = 281,
- FOR_t = 282,
- IF_t = 283,
- IGNORE_t = 284,
- IN_t = 285,
- INCLUDE_t = 286,
- LOCAL_t = 287,
- MODULE_t = 288,
- ON_t = 289,
- PIECEMEAL_t = 290,
- QUIETLY_t = 291,
- RETURN_t = 292,
- RULE_t = 293,
- SWITCH_t = 294,
- TOGETHER_t = 295,
- UPDATED_t = 296,
- WHILE_t = 297,
- _LBRACE_t = 298,
- _BAR_t = 299,
- _BARBAR_t = 300,
- _RBRACE_t = 301,
- ARG = 302,
- STRING = 303
- };
-#endif
-/* Tokens. */
-#define _BANG_t 258
-#define _BANG_EQUALS_t 259
-#define _AMPER_t 260
-#define _AMPERAMPER_t 261
-#define _LPAREN_t 262
-#define _RPAREN_t 263
-#define _PLUS_EQUALS_t 264
-#define _COLON_t 265
-#define _SEMIC_t 266
-#define _LANGLE_t 267
-#define _LANGLE_EQUALS_t 268
-#define _EQUALS_t 269
-#define _RANGLE_t 270
-#define _RANGLE_EQUALS_t 271
-#define _QUESTION_EQUALS_t 272
-#define _LBRACKET_t 273
-#define _RBRACKET_t 274
-#define ACTIONS_t 275
-#define BIND_t 276
-#define CASE_t 277
-#define CLASS_t 278
-#define DEFAULT_t 279
-#define ELSE_t 280
-#define EXISTING_t 281
-#define FOR_t 282
-#define IF_t 283
-#define IGNORE_t 284
-#define IN_t 285
-#define INCLUDE_t 286
-#define LOCAL_t 287
-#define MODULE_t 288
-#define ON_t 289
-#define PIECEMEAL_t 290
-#define QUIETLY_t 291
-#define RETURN_t 292
-#define RULE_t 293
-#define SWITCH_t 294
-#define TOGETHER_t 295
-#define UPDATED_t 296
-#define WHILE_t 297
-#define _LBRACE_t 298
-#define _BAR_t 299
-#define _BARBAR_t 300
-#define _RBRACE_t 301
-#define ARG 302
-#define STRING 303
-
-
-
-
-#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED
-typedef int YYSTYPE;
-# define YYSTYPE_IS_TRIVIAL 1
-# define yystype YYSTYPE /* obsolescent; will be withdrawn */
-# define YYSTYPE_IS_DECLARED 1
-#endif
-
-extern YYSTYPE yylval;
-
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/jamgram.y
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/jamgram.y b/ext/kenlm/jam-files/engine/jamgram.y
deleted file mode 100644
index 543f156..0000000
--- a/ext/kenlm/jam-files/engine/jamgram.y
+++ /dev/null
@@ -1,373 +0,0 @@
-%token _BANG_t
-%token _BANG_EQUALS_t
-%token _AMPER_t
-%token _AMPERAMPER_t
-%token _LPAREN_t
-%token _RPAREN_t
-%token _PLUS_EQUALS_t
-%token _COLON_t
-%token _SEMIC_t
-%token _LANGLE_t
-%token _LANGLE_EQUALS_t
-%token _EQUALS_t
-%token _RANGLE_t
-%token _RANGLE_EQUALS_t
-%token _QUESTION_EQUALS_t
-%token _LBRACKET_t
-%token _RBRACKET_t
-%token ACTIONS_t
-%token BIND_t
-%token CASE_t
-%token CLASS_t
-%token DEFAULT_t
-%token ELSE_t
-%token EXISTING_t
-%token FOR_t
-%token IF_t
-%token IGNORE_t
-%token IN_t
-%token INCLUDE_t
-%token LOCAL_t
-%token MODULE_t
-%token ON_t
-%token PIECEMEAL_t
-%token QUIETLY_t
-%token RETURN_t
-%token RULE_t
-%token SWITCH_t
-%token TOGETHER_t
-%token UPDATED_t
-%token WHILE_t
-%token _LBRACE_t
-%token _BAR_t
-%token _BARBAR_t
-%token _RBRACE_t
-/*
- * Copyright 1993, 2000 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * jamgram.yy - jam grammar
- *
- * 04/13/94 (seiwald) - added shorthand L0 for null list pointer
- * 06/01/94 (seiwald) - new 'actions existing' does existing sources
- * 08/23/94 (seiwald) - Support for '+=' (append to variable)
- * 08/31/94 (seiwald) - Allow ?= as alias for "default =".
- * 09/15/94 (seiwald) - if conditionals take only single arguments, so
- * that 'if foo == bar' gives syntax error (use =).
- * 02/11/95 (seiwald) - when scanning arguments to rules, only treat
- * punctuation keywords as keywords. All arg lists
- * are terminated with punctuation keywords.
- *
- * 09/11/00 (seiwald) - Support for function calls:
- *
- * Rules now return lists (LIST *), rather than void.
- *
- * New "[ rule ]" syntax evals rule into a LIST.
- *
- * Lists are now generated by compile_list() and
- * compile_append(), and any other rule that indirectly
- * makes a list, rather than being built directly here,
- * so that lists values can contain rule evaluations.
- *
- * New 'return' rule sets the return value, though
- * other statements also may have return values.
- *
- * 'run' production split from 'block' production so
- * that empty blocks can be handled separately.
- */
-
-%token ARG STRING
-
-%left _BARBAR_t _BAR_t
-%left _AMPERAMPER_t _AMPER_t
-%left _EQUALS_t _BANG_EQUALS_t IN_t
-%left _LANGLE_t _LANGLE_EQUALS_t _RANGLE_t _RANGLE_EQUALS_t
-%left _BANG_t
-
-%{
-#include "jam.h"
-
-#include "lists.h"
-#include "parse.h"
-#include "scan.h"
-#include "compile.h"
-#include "object.h"
-#include "rules.h"
-
-# define YYMAXDEPTH 10000 /* for OSF and other less endowed yaccs */
-
-# define F0 -1
-# define P0 (PARSE *)0
-# define S0 (OBJECT *)0
-
-# define pappend( l,r ) parse_make( PARSE_APPEND,l,r,P0,S0,S0,0 )
-# define peval( c,l,r ) parse_make( PARSE_EVAL,l,r,P0,S0,S0,c )
-# define pfor( s,l,r,x ) parse_make( PARSE_FOREACH,l,r,P0,s,S0,x )
-# define pif( l,r,t ) parse_make( PARSE_IF,l,r,t,S0,S0,0 )
-# define pincl( l ) parse_make( PARSE_INCLUDE,l,P0,P0,S0,S0,0 )
-# define plist( s ) parse_make( PARSE_LIST,P0,P0,P0,s,S0,0 )
-# define plocal( l,r,t ) parse_make( PARSE_LOCAL,l,r,t,S0,S0,0 )
-# define pmodule( l,r ) parse_make( PARSE_MODULE,l,r,P0,S0,S0,0 )
-# define pclass( l,r ) parse_make( PARSE_CLASS,l,r,P0,S0,S0,0 )
-# define pnull() parse_make( PARSE_NULL,P0,P0,P0,S0,S0,0 )
-# define pon( l,r ) parse_make( PARSE_ON,l,r,P0,S0,S0,0 )
-# define prule( s,p ) parse_make( PARSE_RULE,p,P0,P0,s,S0,0 )
-# define prules( l,r ) parse_make( PARSE_RULES,l,r,P0,S0,S0,0 )
-# define pset( l,r,a ) parse_make( PARSE_SET,l,r,P0,S0,S0,a )
-# define pset1( l,r,t,a ) parse_make( PARSE_SETTINGS,l,r,t,S0,S0,a )
-# define psetc( s,p,a,l ) parse_make( PARSE_SETCOMP,p,a,P0,s,S0,l )
-# define psete( s,l,s1,f ) parse_make( PARSE_SETEXEC,l,P0,P0,s,s1,f )
-# define pswitch( l,r ) parse_make( PARSE_SWITCH,l,r,P0,S0,S0,0 )
-# define pwhile( l,r ) parse_make( PARSE_WHILE,l,r,P0,S0,S0,0 )
-
-# define pnode( l,r ) parse_make( F0,l,r,P0,S0,S0,0 )
-# define psnode( s,l ) parse_make( F0,l,P0,P0,s,S0,0 )
-
-%}
-
-%%
-
-run : /* empty */
- /* do nothing */
- | rules
- { parse_save( $1.parse ); }
- ;
-
-/*
- * block - zero or more rules
- * rules - one or more rules
- * rule - any one of jam's rules
- * right-recursive so rules execute in order.
- */
-
-block : null
- { $$.parse = $1.parse; }
- | rules
- { $$.parse = $1.parse; }
- ;
-
-rules : rule
- { $$.parse = $1.parse; }
- | rule rules
- { $$.parse = prules( $1.parse, $2.parse ); }
- | LOCAL_t list assign_list_opt _SEMIC_t block
- { $$.parse = plocal( $2.parse, $3.parse, $5.parse ); }
- ;
-
-null : /* empty */
- { $$.parse = pnull(); }
- ;
-
-assign_list_opt : _EQUALS_t list
- { $$.parse = $2.parse; $$.number = ASSIGN_SET; }
- | null
- { $$.parse = $1.parse; $$.number = ASSIGN_APPEND; }
- ;
-
-arglist_opt : _LPAREN_t lol _RPAREN_t
- { $$.parse = $2.parse; }
- |
- { $$.parse = P0; }
- ;
-
-local_opt : LOCAL_t
- { $$.number = 1; }
- | /* empty */
- { $$.number = 0; }
- ;
-
-rule : _LBRACE_t block _RBRACE_t
- { $$.parse = $2.parse; }
- | INCLUDE_t list _SEMIC_t
- { $$.parse = pincl( $2.parse ); }
- | ARG lol _SEMIC_t
- { $$.parse = prule( $1.string, $2.parse ); }
- | arg assign list _SEMIC_t
- { $$.parse = pset( $1.parse, $3.parse, $2.number ); }
- | arg ON_t list assign list _SEMIC_t
- { $$.parse = pset1( $1.parse, $3.parse, $5.parse, $4.number ); }
- | RETURN_t list _SEMIC_t
- { $$.parse = $2.parse; }
- | FOR_t local_opt ARG IN_t list _LBRACE_t block _RBRACE_t
- { $$.parse = pfor( $3.string, $5.parse, $7.parse, $2.number ); }
- | SWITCH_t list _LBRACE_t cases _RBRACE_t
- { $$.parse = pswitch( $2.parse, $4.parse ); }
- | IF_t expr _LBRACE_t block _RBRACE_t
- { $$.parse = pif( $2.parse, $4.parse, pnull() ); }
- | MODULE_t list _LBRACE_t block _RBRACE_t
- { $$.parse = pmodule( $2.parse, $4.parse ); }
- | CLASS_t lol _LBRACE_t block _RBRACE_t
- { $$.parse = pclass( $2.parse, $4.parse ); }
- | WHILE_t expr _LBRACE_t block _RBRACE_t
- { $$.parse = pwhile( $2.parse, $4.parse ); }
- | IF_t expr _LBRACE_t block _RBRACE_t ELSE_t rule
- { $$.parse = pif( $2.parse, $4.parse, $7.parse ); }
- | local_opt RULE_t ARG arglist_opt rule
- { $$.parse = psetc( $3.string, $5.parse, $4.parse, $1.number ); }
- | ON_t arg rule
- { $$.parse = pon( $2.parse, $3.parse ); }
- | ACTIONS_t eflags ARG bindlist _LBRACE_t
- { yymode( SCAN_STRING ); }
- STRING
- { yymode( SCAN_NORMAL ); }
- _RBRACE_t
- { $$.parse = psete( $3.string,$4.parse,$7.string,$2.number ); }
- ;
-
-/*
- * assign - = or +=
- */
-
-assign : _EQUALS_t
- { $$.number = ASSIGN_SET; }
- | _PLUS_EQUALS_t
- { $$.number = ASSIGN_APPEND; }
- | _QUESTION_EQUALS_t
- { $$.number = ASSIGN_DEFAULT; }
- | DEFAULT_t _EQUALS_t
- { $$.number = ASSIGN_DEFAULT; }
- ;
-
-/*
- * expr - an expression for if
- */
-expr : arg
- { $$.parse = peval( EXPR_EXISTS, $1.parse, pnull() ); }
- | expr _EQUALS_t expr
- { $$.parse = peval( EXPR_EQUALS, $1.parse, $3.parse ); }
- | expr _BANG_EQUALS_t expr
- { $$.parse = peval( EXPR_NOTEQ, $1.parse, $3.parse ); }
- | expr _LANGLE_t expr
- { $$.parse = peval( EXPR_LESS, $1.parse, $3.parse ); }
- | expr _LANGLE_EQUALS_t expr
- { $$.parse = peval( EXPR_LESSEQ, $1.parse, $3.parse ); }
- | expr _RANGLE_t expr
- { $$.parse = peval( EXPR_MORE, $1.parse, $3.parse ); }
- | expr _RANGLE_EQUALS_t expr
- { $$.parse = peval( EXPR_MOREEQ, $1.parse, $3.parse ); }
- | expr _AMPER_t expr
- { $$.parse = peval( EXPR_AND, $1.parse, $3.parse ); }
- | expr _AMPERAMPER_t expr
- { $$.parse = peval( EXPR_AND, $1.parse, $3.parse ); }
- | expr _BAR_t expr
- { $$.parse = peval( EXPR_OR, $1.parse, $3.parse ); }
- | expr _BARBAR_t expr
- { $$.parse = peval( EXPR_OR, $1.parse, $3.parse ); }
- | arg IN_t list
- { $$.parse = peval( EXPR_IN, $1.parse, $3.parse ); }
- | _BANG_t expr
- { $$.parse = peval( EXPR_NOT, $2.parse, pnull() ); }
- | _LPAREN_t expr _RPAREN_t
- { $$.parse = $2.parse; }
- ;
-
-
-/*
- * cases - action elements inside a 'switch'
- * case - a single action element inside a 'switch'
- * right-recursive rule so cases can be examined in order.
- */
-
-cases : /* empty */
- { $$.parse = P0; }
- | case cases
- { $$.parse = pnode( $1.parse, $2.parse ); }
- ;
-
-case : CASE_t ARG _COLON_t block
- { $$.parse = psnode( $2.string, $4.parse ); }
- ;
-
-/*
- * lol - list of lists
- * right-recursive rule so that lists can be added in order.
- */
-
-lol : list
- { $$.parse = pnode( P0, $1.parse ); }
- | list _COLON_t lol
- { $$.parse = pnode( $3.parse, $1.parse ); }
- ;
-
-/*
- * list - zero or more args in a LIST
- * listp - list (in puncutation only mode)
- * arg - one ARG or function call
- */
-
-list : listp
- { $$.parse = $1.parse; yymode( SCAN_NORMAL ); }
- ;
-
-listp : /* empty */
- { $$.parse = pnull(); yymode( SCAN_PUNCT ); }
- | listp arg
- { $$.parse = pappend( $1.parse, $2.parse ); }
- ;
-
-arg : ARG
- { $$.parse = plist( $1.string ); }
- | _LBRACKET_t { yymode( SCAN_NORMAL ); } func _RBRACKET_t
- { $$.parse = $3.parse; }
- ;
-
-/*
- * func - a function call (inside [])
- * This needs to be split cleanly out of 'rule'
- */
-
-func : ARG lol
- { $$.parse = prule( $1.string, $2.parse ); }
- | ON_t arg ARG lol
- { $$.parse = pon( $2.parse, prule( $3.string, $4.parse ) ); }
- | ON_t arg RETURN_t list
- { $$.parse = pon( $2.parse, $4.parse ); }
- ;
-
-
-/*
- * eflags - zero or more modifiers to 'executes'
- * eflag - a single modifier to 'executes'
- */
-
-eflags : /* empty */
- { $$.number = 0; }
- | eflags eflag
- { $$.number = $1.number | $2.number; }
- ;
-
-eflag : UPDATED_t
- { $$.number = EXEC_UPDATED; }
- | TOGETHER_t
- { $$.number = EXEC_TOGETHER; }
- | IGNORE_t
- { $$.number = EXEC_IGNORE; }
- | QUIETLY_t
- { $$.number = EXEC_QUIETLY; }
- | PIECEMEAL_t
- { $$.number = EXEC_PIECEMEAL; }
- | EXISTING_t
- { $$.number = EXEC_EXISTING; }
- ;
-
-
-/*
- * bindlist - list of variable to bind for an action
- */
-
-bindlist : /* empty */
- { $$.parse = pnull(); }
- | BIND_t list
- { $$.parse = $2.parse; }
- ;
-
-
[08/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/search_trie.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/search_trie.cc b/ext/kenlm/lm/search_trie.cc
deleted file mode 100644
index a63985a..0000000
--- a/ext/kenlm/lm/search_trie.cc
+++ /dev/null
@@ -1,600 +0,0 @@
-/* This is where the trie is built. It's on-disk. */
-#include "lm/search_trie.hh"
-
-#include "lm/bhiksha.hh"
-#include "lm/binary_format.hh"
-#include "lm/blank.hh"
-#include "lm/lm_exception.hh"
-#include "lm/max_order.hh"
-#include "lm/quantize.hh"
-#include "lm/trie.hh"
-#include "lm/trie_sort.hh"
-#include "lm/vocab.hh"
-#include "lm/weights.hh"
-#include "lm/word_index.hh"
-#include "util/ersatz_progress.hh"
-#include "util/mmap.hh"
-#include "util/proxy_iterator.hh"
-#include "util/scoped.hh"
-#include "util/sized_iterator.hh"
-
-#include <algorithm>
-#include <cstring>
-#include <cstdio>
-#include <cstdlib>
-#include <queue>
-#include <limits>
-#include <numeric>
-#include <vector>
-
-#if defined(_WIN32) || defined(_WIN64)
-#include <windows.h>
-#endif
-
-namespace lm {
-namespace ngram {
-namespace trie {
-namespace {
-
-void ReadOrThrow(FILE *from, void *data, size_t size) {
- UTIL_THROW_IF(1 != std::fread(data, size, 1, from), util::ErrnoException, "Short read");
-}
-
-int Compare(unsigned char order, const void *first_void, const void *second_void) {
- const WordIndex *first = reinterpret_cast<const WordIndex*>(first_void), *second = reinterpret_cast<const WordIndex*>(second_void);
- const WordIndex *end = first + order;
- for (; first != end; ++first, ++second) {
- if (*first < *second) return -1;
- if (*first > *second) return 1;
- }
- return 0;
-}
-
-struct ProbPointer {
- unsigned char array;
- uint64_t index;
-};
-
-// Array of n-grams and float indices.
-class BackoffMessages {
- public:
- void Init(std::size_t entry_size) {
- current_ = NULL;
- allocated_ = NULL;
- entry_size_ = entry_size;
- }
-
- void Add(const WordIndex *to, ProbPointer index) {
- while (current_ + entry_size_ > allocated_) {
- std::size_t allocated_size = allocated_ - (uint8_t*)backing_.get();
- Resize(std::max<std::size_t>(allocated_size * 2, entry_size_));
- }
- memcpy(current_, to, entry_size_ - sizeof(ProbPointer));
- *reinterpret_cast<ProbPointer*>(current_ + entry_size_ - sizeof(ProbPointer)) = index;
- current_ += entry_size_;
- }
-
- void Apply(float *const *const base, FILE *unigrams) {
- FinishedAdding();
- if (current_ == allocated_) return;
- rewind(unigrams);
- ProbBackoff weights;
- WordIndex unigram = 0;
- ReadOrThrow(unigrams, &weights, sizeof(weights));
- for (; current_ != allocated_; current_ += entry_size_) {
- const WordIndex &cur_word = *reinterpret_cast<const WordIndex*>(current_);
- for (; unigram < cur_word; ++unigram) {
- ReadOrThrow(unigrams, &weights, sizeof(weights));
- }
- if (!HasExtension(weights.backoff)) {
- weights.backoff = kExtensionBackoff;
- UTIL_THROW_IF(fseek(unigrams, -sizeof(weights), SEEK_CUR), util::ErrnoException, "Seeking backwards to denote unigram extension failed.");
- util::WriteOrThrow(unigrams, &weights, sizeof(weights));
- }
- const ProbPointer &write_to = *reinterpret_cast<const ProbPointer*>(current_ + sizeof(WordIndex));
- base[write_to.array][write_to.index] += weights.backoff;
- }
- backing_.reset();
- }
-
- void Apply(float *const *const base, RecordReader &reader) {
- FinishedAdding();
- if (current_ == allocated_) return;
- // We'll also use the same buffer to record messages to blanks that they extend.
- WordIndex *extend_out = reinterpret_cast<WordIndex*>(current_);
- const unsigned char order = (entry_size_ - sizeof(ProbPointer)) / sizeof(WordIndex);
- for (reader.Rewind(); reader && (current_ != allocated_); ) {
- switch (Compare(order, reader.Data(), current_)) {
- case -1:
- ++reader;
- break;
- case 1:
- // Message but nobody to receive it. Write it down at the beginning of the buffer so we can inform this blank that it extends.
- for (const WordIndex *w = reinterpret_cast<const WordIndex *>(current_); w != reinterpret_cast<const WordIndex *>(current_) + order; ++w, ++extend_out) *extend_out = *w;
- current_ += entry_size_;
- break;
- case 0:
- float &backoff = reinterpret_cast<ProbBackoff*>((uint8_t*)reader.Data() + order * sizeof(WordIndex))->backoff;
- if (!HasExtension(backoff)) {
- backoff = kExtensionBackoff;
- reader.Overwrite(&backoff, sizeof(float));
- } else {
- const ProbPointer &write_to = *reinterpret_cast<const ProbPointer*>(current_ + entry_size_ - sizeof(ProbPointer));
- base[write_to.array][write_to.index] += backoff;
- }
- current_ += entry_size_;
- break;
- }
- }
- // Now this is a list of blanks that extend right.
- entry_size_ = sizeof(WordIndex) * order;
- Resize(sizeof(WordIndex) * (extend_out - (const WordIndex*)backing_.get()));
- current_ = (uint8_t*)backing_.get();
- }
-
- // Call after Apply
- bool Extends(unsigned char order, const WordIndex *words) {
- if (current_ == allocated_) return false;
- assert(order * sizeof(WordIndex) == entry_size_);
- while (true) {
- switch(Compare(order, words, current_)) {
- case 1:
- current_ += entry_size_;
- if (current_ == allocated_) return false;
- break;
- case -1:
- return false;
- case 0:
- return true;
- }
- }
- }
-
- private:
- void FinishedAdding() {
- Resize(current_ - (uint8_t*)backing_.get());
- // Sort requests in same order as files.
- std::sort(
- util::SizedIterator(util::SizedProxy(backing_.get(), entry_size_)),
- util::SizedIterator(util::SizedProxy(current_, entry_size_)),
- util::SizedCompare<EntryCompare>(EntryCompare((entry_size_ - sizeof(ProbPointer)) / sizeof(WordIndex))));
- current_ = (uint8_t*)backing_.get();
- }
-
- void Resize(std::size_t to) {
- std::size_t current = current_ - (uint8_t*)backing_.get();
- backing_.call_realloc(to);
- current_ = (uint8_t*)backing_.get() + current;
- allocated_ = (uint8_t*)backing_.get() + to;
- }
-
- util::scoped_malloc backing_;
-
- uint8_t *current_, *allocated_;
-
- std::size_t entry_size_;
-};
-
-const float kBadProb = std::numeric_limits<float>::infinity();
-
-class SRISucks {
- public:
- SRISucks() {
- for (BackoffMessages *i = messages_; i != messages_ + KENLM_MAX_ORDER - 1; ++i)
- i->Init(sizeof(ProbPointer) + sizeof(WordIndex) * (i - messages_ + 1));
- }
-
- void Send(unsigned char begin, unsigned char order, const WordIndex *to, float prob_basis) {
- assert(prob_basis != kBadProb);
- ProbPointer pointer;
- pointer.array = order - 1;
- pointer.index = values_[order - 1].size();
- for (unsigned char i = begin; i < order; ++i) {
- messages_[i - 1].Add(to, pointer);
- }
- values_[order - 1].push_back(prob_basis);
- }
-
- void ObtainBackoffs(unsigned char total_order, FILE *unigram_file, RecordReader *reader) {
- for (unsigned char i = 0; i < KENLM_MAX_ORDER - 1; ++i) {
- it_[i] = values_[i].empty() ? NULL : &*values_[i].begin();
- }
- messages_[0].Apply(it_, unigram_file);
- BackoffMessages *messages = messages_ + 1;
- const RecordReader *end = reader + total_order - 2 /* exclude unigrams and longest order */;
- for (; reader != end; ++messages, ++reader) {
- messages->Apply(it_, *reader);
- }
- }
-
- ProbBackoff GetBlank(unsigned char total_order, unsigned char order, const WordIndex *indices) {
- assert(order > 1);
- ProbBackoff ret;
- ret.prob = *(it_[order - 1]++);
- ret.backoff = ((order != total_order - 1) && messages_[order - 1].Extends(order, indices)) ? kExtensionBackoff : kNoExtensionBackoff;
- return ret;
- }
-
- const std::vector<float> &Values(unsigned char order) const {
- return values_[order - 1];
- }
-
- private:
- // This used to be one array. Then I needed to separate it by order for quantization to work.
- std::vector<float> values_[KENLM_MAX_ORDER - 1];
- BackoffMessages messages_[KENLM_MAX_ORDER - 1];
-
- float *it_[KENLM_MAX_ORDER - 1];
-};
-
-class FindBlanks {
- public:
- FindBlanks(unsigned char order, const ProbBackoff *unigrams, SRISucks &messages)
- : counts_(order), unigrams_(unigrams), sri_(messages) {}
-
- float UnigramProb(WordIndex index) const {
- return unigrams_[index].prob;
- }
-
- void Unigram(WordIndex /*index*/) {
- ++counts_[0];
- }
-
- void MiddleBlank(const unsigned char order, const WordIndex *indices, unsigned char lower, float prob_basis) {
- sri_.Send(lower, order, indices + 1, prob_basis);
- ++counts_[order - 1];
- }
-
- void Middle(const unsigned char order, const void * /*data*/) {
- ++counts_[order - 1];
- }
-
- void Longest(const void * /*data*/) {
- ++counts_.back();
- }
-
- const std::vector<uint64_t> &Counts() const {
- return counts_;
- }
-
- private:
- std::vector<uint64_t> counts_;
-
- const ProbBackoff *unigrams_;
-
- SRISucks &sri_;
-};
-
-// Phase to actually write n-grams to the trie.
-template <class Quant, class Bhiksha> class WriteEntries {
- public:
- WriteEntries(RecordReader *contexts, const Quant &quant, UnigramValue *unigrams, BitPackedMiddle<Bhiksha> *middle, BitPackedLongest &longest, unsigned char order, SRISucks &sri) :
- contexts_(contexts),
- quant_(quant),
- unigrams_(unigrams),
- middle_(middle),
- longest_(longest),
- bigram_pack_((order == 2) ? static_cast<BitPacked&>(longest_) : static_cast<BitPacked&>(*middle_)),
- order_(order),
- sri_(sri) {}
-
- float UnigramProb(WordIndex index) const { return unigrams_[index].weights.prob; }
-
- void Unigram(WordIndex word) {
- unigrams_[word].next = bigram_pack_.InsertIndex();
- }
-
- void MiddleBlank(const unsigned char order, const WordIndex *indices, unsigned char /*lower*/, float /*prob_base*/) {
- ProbBackoff weights = sri_.GetBlank(order_, order, indices);
- typename Quant::MiddlePointer(quant_, order - 2, middle_[order - 2].Insert(indices[order - 1])).Write(weights.prob, weights.backoff);
- }
-
- void Middle(const unsigned char order, const void *data) {
- RecordReader &context = contexts_[order - 1];
- const WordIndex *words = reinterpret_cast<const WordIndex*>(data);
- ProbBackoff weights = *reinterpret_cast<const ProbBackoff*>(words + order);
- if (context && !memcmp(data, context.Data(), sizeof(WordIndex) * order)) {
- SetExtension(weights.backoff);
- ++context;
- }
- typename Quant::MiddlePointer(quant_, order - 2, middle_[order - 2].Insert(words[order - 1])).Write(weights.prob, weights.backoff);
- }
-
- void Longest(const void *data) {
- const WordIndex *words = reinterpret_cast<const WordIndex*>(data);
- typename Quant::LongestPointer(quant_, longest_.Insert(words[order_ - 1])).Write(reinterpret_cast<const Prob*>(words + order_)->prob);
- }
-
- private:
- RecordReader *contexts_;
- const Quant &quant_;
- UnigramValue *const unigrams_;
- BitPackedMiddle<Bhiksha> *const middle_;
- BitPackedLongest &longest_;
- BitPacked &bigram_pack_;
- const unsigned char order_;
- SRISucks &sri_;
-};
-
-struct Gram {
- Gram(const WordIndex *in_begin, unsigned char order) : begin(in_begin), end(in_begin + order) {}
-
- const WordIndex *begin, *end;
-
- // For queue, this is the direction we want.
- bool operator<(const Gram &other) const {
- return std::lexicographical_compare(other.begin, other.end, begin, end);
- }
-};
-
-template <class Doing> class BlankManager {
- public:
- BlankManager(unsigned char total_order, Doing &doing) : total_order_(total_order), been_length_(0), doing_(doing) {
- for (float *i = basis_; i != basis_ + KENLM_MAX_ORDER - 1; ++i) *i = kBadProb;
- }
-
- void Visit(const WordIndex *to, unsigned char length, float prob) {
- basis_[length - 1] = prob;
- unsigned char overlap = std::min<unsigned char>(length - 1, been_length_);
- const WordIndex *cur;
- WordIndex *pre;
- for (cur = to, pre = been_; cur != to + overlap; ++cur, ++pre) {
- if (*pre != *cur) break;
- }
- if (cur == to + length - 1) {
- *pre = *cur;
- been_length_ = length;
- return;
- }
- // There are blanks to insert starting with order blank.
- unsigned char blank = cur - to + 1;
- UTIL_THROW_IF(blank == 1, FormatLoadException, "Missing a unigram that appears as context.");
- const float *lower_basis;
- for (lower_basis = basis_ + blank - 2; *lower_basis == kBadProb; --lower_basis) {}
- unsigned char based_on = lower_basis - basis_ + 1;
- for (; cur != to + length - 1; ++blank, ++cur, ++pre) {
- assert(*lower_basis != kBadProb);
- doing_.MiddleBlank(blank, to, based_on, *lower_basis);
- *pre = *cur;
- // Mark that the probability is a blank so it shouldn't be used as the basis for a later n-gram.
- basis_[blank - 1] = kBadProb;
- }
- *pre = *cur;
- been_length_ = length;
- }
-
- private:
- const unsigned char total_order_;
-
- WordIndex been_[KENLM_MAX_ORDER];
- unsigned char been_length_;
-
- float basis_[KENLM_MAX_ORDER];
-
- Doing &doing_;
-};
-
-template <class Doing> void RecursiveInsert(const unsigned char total_order, const WordIndex unigram_count, RecordReader *input, std::ostream *progress_out, const char *message, Doing &doing) {
- util::ErsatzProgress progress(unigram_count + 1, progress_out, message);
- WordIndex unigram = 0;
- std::priority_queue<Gram> grams;
- if (unigram_count) grams.push(Gram(&unigram, 1));
- for (unsigned char i = 2; i <= total_order; ++i) {
- if (input[i-2]) grams.push(Gram(reinterpret_cast<const WordIndex*>(input[i-2].Data()), i));
- }
-
- BlankManager<Doing> blank(total_order, doing);
-
- while (!grams.empty()) {
- Gram top = grams.top();
- grams.pop();
- unsigned char order = top.end - top.begin;
- if (order == 1) {
- blank.Visit(&unigram, 1, doing.UnigramProb(unigram));
- doing.Unigram(unigram);
- progress.Set(unigram);
- if (++unigram < unigram_count) grams.push(top);
- } else {
- if (order == total_order) {
- blank.Visit(top.begin, order, reinterpret_cast<const Prob*>(top.end)->prob);
- doing.Longest(top.begin);
- } else {
- blank.Visit(top.begin, order, reinterpret_cast<const ProbBackoff*>(top.end)->prob);
- doing.Middle(order, top.begin);
- }
- RecordReader &reader = input[order - 2];
- if (++reader) grams.push(top);
- }
- }
-}
-
-void SanityCheckCounts(const std::vector<uint64_t> &initial, const std::vector<uint64_t> &fixed) {
- if (fixed[0] != initial[0]) UTIL_THROW(util::Exception, "Unigram count should be constant but initial is " << initial[0] << " and recounted is " << fixed[0]);
- if (fixed.back() != initial.back()) UTIL_THROW(util::Exception, "Longest count should be constant but it changed from " << initial.back() << " to " << fixed.back());
- for (unsigned char i = 0; i < initial.size(); ++i) {
- if (fixed[i] < initial[i]) UTIL_THROW(util::Exception, "Counts came out lower than expected. This shouldn't happen");
- }
-}
-
-template <class Quant> void TrainQuantizer(uint8_t order, uint64_t count, const std::vector<float> &additional, RecordReader &reader, util::ErsatzProgress &progress, Quant &quant) {
- std::vector<float> probs(additional), backoffs;
- probs.reserve(count + additional.size());
- backoffs.reserve(count);
- for (reader.Rewind(); reader; ++reader) {
- const ProbBackoff &weights = *reinterpret_cast<const ProbBackoff*>(reinterpret_cast<const uint8_t*>(reader.Data()) + sizeof(WordIndex) * order);
- probs.push_back(weights.prob);
- if (weights.backoff != 0.0) backoffs.push_back(weights.backoff);
- ++progress;
- }
- quant.Train(order, probs, backoffs);
-}
-
-template <class Quant> void TrainProbQuantizer(uint8_t order, uint64_t count, RecordReader &reader, util::ErsatzProgress &progress, Quant &quant) {
- std::vector<float> probs, backoffs;
- probs.reserve(count);
- for (reader.Rewind(); reader; ++reader) {
- const Prob &weights = *reinterpret_cast<const Prob*>(reinterpret_cast<const uint8_t*>(reader.Data()) + sizeof(WordIndex) * order);
- probs.push_back(weights.prob);
- ++progress;
- }
- quant.TrainProb(order, probs);
-}
-
-void PopulateUnigramWeights(FILE *file, WordIndex unigram_count, RecordReader &contexts, UnigramValue *unigrams) {
- // Fill unigram probabilities.
- try {
- rewind(file);
- for (WordIndex i = 0; i < unigram_count; ++i) {
- ReadOrThrow(file, &unigrams[i].weights, sizeof(ProbBackoff));
- if (contexts && *reinterpret_cast<const WordIndex*>(contexts.Data()) == i) {
- SetExtension(unigrams[i].weights.backoff);
- ++contexts;
- }
- }
- } catch (util::Exception &e) {
- e << " while re-reading unigram probabilities";
- throw;
- }
-}
-
-} // namespace
-
-template <class Quant, class Bhiksha> void BuildTrie(SortedFiles &files, std::vector<uint64_t> &counts, const Config &config, TrieSearch<Quant, Bhiksha> &out, Quant &quant, SortedVocabulary &vocab, BinaryFormat &backing) {
- RecordReader inputs[KENLM_MAX_ORDER - 1];
- RecordReader contexts[KENLM_MAX_ORDER - 1];
-
- for (unsigned char i = 2; i <= counts.size(); ++i) {
- inputs[i-2].Init(files.Full(i), i * sizeof(WordIndex) + (i == counts.size() ? sizeof(Prob) : sizeof(ProbBackoff)));
- contexts[i-2].Init(files.Context(i), (i-1) * sizeof(WordIndex));
- }
-
- SRISucks sri;
- std::vector<uint64_t> fixed_counts;
- util::scoped_FILE unigram_file;
- util::scoped_fd unigram_fd(files.StealUnigram());
- {
- util::scoped_memory unigrams;
- MapRead(util::POPULATE_OR_READ, unigram_fd.get(), 0, counts[0] * sizeof(ProbBackoff), unigrams);
- FindBlanks finder(counts.size(), reinterpret_cast<const ProbBackoff*>(unigrams.get()), sri);
- RecursiveInsert(counts.size(), counts[0], inputs, config.ProgressMessages(), "Identifying n-grams omitted by SRI", finder);
- fixed_counts = finder.Counts();
- }
- unigram_file.reset(util::FDOpenOrThrow(unigram_fd));
- for (const RecordReader *i = inputs; i != inputs + counts.size() - 2; ++i) {
- if (*i) UTIL_THROW(FormatLoadException, "There's a bug in the trie implementation: the " << (i - inputs + 2) << "-gram table did not complete reading");
- }
- SanityCheckCounts(counts, fixed_counts);
- counts = fixed_counts;
-
- sri.ObtainBackoffs(counts.size(), unigram_file.get(), inputs);
-
- void *vocab_relocate;
- void *search_base = backing.GrowForSearch(TrieSearch<Quant, Bhiksha>::Size(fixed_counts, config), vocab.UnkCountChangePadding(), vocab_relocate);
- vocab.Relocate(vocab_relocate);
- out.SetupMemory(reinterpret_cast<uint8_t*>(search_base), fixed_counts, config);
-
- for (unsigned char i = 2; i <= counts.size(); ++i) {
- inputs[i-2].Rewind();
- }
- if (Quant::kTrain) {
- util::ErsatzProgress progress(std::accumulate(counts.begin() + 1, counts.end(), 0),
- config.ProgressMessages(), "Quantizing");
- for (unsigned char i = 2; i < counts.size(); ++i) {
- TrainQuantizer(i, counts[i-1], sri.Values(i), inputs[i-2], progress, quant);
- }
- TrainProbQuantizer(counts.size(), counts.back(), inputs[counts.size() - 2], progress, quant);
- quant.FinishedLoading(config);
- }
-
- UnigramValue *unigrams = out.unigram_.Raw();
- PopulateUnigramWeights(unigram_file.get(), counts[0], contexts[0], unigrams);
- unigram_file.reset();
-
- for (unsigned char i = 2; i <= counts.size(); ++i) {
- inputs[i-2].Rewind();
- }
- // Fill entries except unigram probabilities.
- {
- WriteEntries<Quant, Bhiksha> writer(contexts, quant, unigrams, out.middle_begin_, out.longest_, counts.size(), sri);
- RecursiveInsert(counts.size(), counts[0], inputs, config.ProgressMessages(), "Writing trie", writer);
- // Write the last unigram entry, which is the end pointer for the bigrams.
- writer.Unigram(counts[0]);
- }
-
- // Do not disable this error message or else too little state will be returned. Both WriteEntries::Middle and returning state based on found n-grams will need to be fixed to handle this situation.
- for (unsigned char order = 2; order <= counts.size(); ++order) {
- const RecordReader &context = contexts[order - 2];
- if (context) {
- FormatLoadException e;
- e << "A " << static_cast<unsigned int>(order) << "-gram has context";
- const WordIndex *ctx = reinterpret_cast<const WordIndex*>(context.Data());
- for (const WordIndex *i = ctx; i != ctx + order - 1; ++i) {
- e << ' ' << *i;
- }
- e << " so this context must appear in the model as a " << static_cast<unsigned int>(order - 1) << "-gram but it does not";
- throw e;
- }
- }
-
- /* Set ending offsets so the last entry will be sized properly */
- // Last entry for unigrams was already set.
- if (out.middle_begin_ != out.middle_end_) {
- for (typename TrieSearch<Quant, Bhiksha>::Middle *i = out.middle_begin_; i != out.middle_end_ - 1; ++i) {
- i->FinishedLoading((i+1)->InsertIndex(), config);
- }
- (out.middle_end_ - 1)->FinishedLoading(out.longest_.InsertIndex(), config);
- }
-}
-
-template <class Quant, class Bhiksha> uint8_t *TrieSearch<Quant, Bhiksha>::SetupMemory(uint8_t *start, const std::vector<uint64_t> &counts, const Config &config) {
- quant_.SetupMemory(start, counts.size(), config);
- start += Quant::Size(counts.size(), config);
- unigram_.Init(start);
- start += Unigram::Size(counts[0]);
- FreeMiddles();
- middle_begin_ = static_cast<Middle*>(malloc(sizeof(Middle) * (counts.size() - 2)));
- middle_end_ = middle_begin_ + (counts.size() - 2);
- std::vector<uint8_t*> middle_starts(counts.size() - 2);
- for (unsigned char i = 2; i < counts.size(); ++i) {
- middle_starts[i-2] = start;
- start += Middle::Size(Quant::MiddleBits(config), counts[i-1], counts[0], counts[i], config);
- }
- // Crazy backwards thing so we initialize using pointers to ones that have already been initialized
- for (unsigned char i = counts.size() - 1; i >= 2; --i) {
- // use "placement new" syntax to initalize Middle in an already-allocated memory location
- new (middle_begin_ + i - 2) Middle(
- middle_starts[i-2],
- quant_.MiddleBits(config),
- counts[i-1],
- counts[0],
- counts[i],
- (i == counts.size() - 1) ? static_cast<const BitPacked&>(longest_) : static_cast<const BitPacked &>(middle_begin_[i-1]),
- config);
- }
- longest_.Init(start, quant_.LongestBits(config), counts[0]);
- return start + Longest::Size(Quant::LongestBits(config), counts.back(), counts[0]);
-}
-
-template <class Quant, class Bhiksha> void TrieSearch<Quant, Bhiksha>::InitializeFromARPA(const char *file, util::FilePiece &f, std::vector<uint64_t> &counts, const Config &config, SortedVocabulary &vocab, BinaryFormat &backing) {
- std::string temporary_prefix;
- if (!config.temporary_directory_prefix.empty()) {
- temporary_prefix = config.temporary_directory_prefix;
- } else if (config.write_mmap) {
- temporary_prefix = config.write_mmap;
- } else {
- temporary_prefix = file;
- }
- // At least 1MB sorting memory.
- SortedFiles sorted(config, f, counts, std::max<size_t>(config.building_memory, 1048576), temporary_prefix, vocab);
-
- BuildTrie(sorted, counts, config, *this, quant_, vocab, backing);
-}
-
-template class TrieSearch<DontQuantize, DontBhiksha>;
-template class TrieSearch<DontQuantize, ArrayBhiksha>;
-template class TrieSearch<SeparatelyQuantize, DontBhiksha>;
-template class TrieSearch<SeparatelyQuantize, ArrayBhiksha>;
-
-} // namespace trie
-} // namespace ngram
-} // namespace lm
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/search_trie.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/search_trie.hh b/ext/kenlm/lm/search_trie.hh
deleted file mode 100644
index 1adba6e..0000000
--- a/ext/kenlm/lm/search_trie.hh
+++ /dev/null
@@ -1,129 +0,0 @@
-#ifndef LM_SEARCH_TRIE_H
-#define LM_SEARCH_TRIE_H
-
-#include "lm/config.hh"
-#include "lm/model_type.hh"
-#include "lm/return.hh"
-#include "lm/trie.hh"
-#include "lm/weights.hh"
-
-#include "util/file.hh"
-#include "util/file_piece.hh"
-
-#include <vector>
-#include <cstdlib>
-#include <cassert>
-
-namespace lm {
-namespace ngram {
-class BinaryFormat;
-class SortedVocabulary;
-namespace trie {
-
-template <class Quant, class Bhiksha> class TrieSearch;
-class SortedFiles;
-template <class Quant, class Bhiksha> void BuildTrie(SortedFiles &files, std::vector<uint64_t> &counts, const Config &config, TrieSearch<Quant, Bhiksha> &out, Quant &quant, SortedVocabulary &vocab, BinaryFormat &backing);
-
-template <class Quant, class Bhiksha> class TrieSearch {
- public:
- typedef NodeRange Node;
-
- typedef ::lm::ngram::trie::UnigramPointer UnigramPointer;
- typedef typename Quant::MiddlePointer MiddlePointer;
- typedef typename Quant::LongestPointer LongestPointer;
-
- static const bool kDifferentRest = false;
-
- static const ModelType kModelType = static_cast<ModelType>(TRIE_SORTED + Quant::kModelTypeAdd + Bhiksha::kModelTypeAdd);
-
- static const unsigned int kVersion = 1;
-
- static void UpdateConfigFromBinary(const BinaryFormat &file, const std::vector<uint64_t> &counts, uint64_t offset, Config &config) {
- Quant::UpdateConfigFromBinary(file, offset, config);
- // Currently the unigram pointers are not compresssed, so there will only be a header for order > 2.
- if (counts.size() > 2)
- Bhiksha::UpdateConfigFromBinary(file, offset + Quant::Size(counts.size(), config) + Unigram::Size(counts[0]), config);
- }
-
- static uint64_t Size(const std::vector<uint64_t> &counts, const Config &config) {
- uint64_t ret = Quant::Size(counts.size(), config) + Unigram::Size(counts[0]);
- for (unsigned char i = 1; i < counts.size() - 1; ++i) {
- ret += Middle::Size(Quant::MiddleBits(config), counts[i], counts[0], counts[i+1], config);
- }
- return ret + Longest::Size(Quant::LongestBits(config), counts.back(), counts[0]);
- }
-
- TrieSearch() : middle_begin_(NULL), middle_end_(NULL) {}
-
- ~TrieSearch() { FreeMiddles(); }
-
- uint8_t *SetupMemory(uint8_t *start, const std::vector<uint64_t> &counts, const Config &config);
-
- void InitializeFromARPA(const char *file, util::FilePiece &f, std::vector<uint64_t> &counts, const Config &config, SortedVocabulary &vocab, BinaryFormat &backing);
-
- unsigned char Order() const {
- return middle_end_ - middle_begin_ + 2;
- }
-
- ProbBackoff &UnknownUnigram() { return unigram_.Unknown(); }
-
- UnigramPointer LookupUnigram(WordIndex word, Node &next, bool &independent_left, uint64_t &extend_left) const {
- extend_left = static_cast<uint64_t>(word);
- UnigramPointer ret(unigram_.Find(word, next));
- independent_left = (next.begin == next.end);
- return ret;
- }
-
- MiddlePointer Unpack(uint64_t extend_pointer, unsigned char extend_length, Node &node) const {
- return MiddlePointer(quant_, extend_length - 2, middle_begin_[extend_length - 2].ReadEntry(extend_pointer, node));
- }
-
- MiddlePointer LookupMiddle(unsigned char order_minus_2, WordIndex word, Node &node, bool &independent_left, uint64_t &extend_left) const {
- util::BitAddress address(middle_begin_[order_minus_2].Find(word, node, extend_left));
- independent_left = (address.base == NULL) || (node.begin == node.end);
- return MiddlePointer(quant_, order_minus_2, address);
- }
-
- LongestPointer LookupLongest(WordIndex word, const Node &node) const {
- return LongestPointer(quant_, longest_.Find(word, node));
- }
-
- bool FastMakeNode(const WordIndex *begin, const WordIndex *end, Node &node) const {
- assert(begin != end);
- bool independent_left;
- uint64_t ignored;
- LookupUnigram(*begin, node, independent_left, ignored);
- for (const WordIndex *i = begin + 1; i < end; ++i) {
- if (independent_left || !LookupMiddle(i - begin - 1, *i, node, independent_left, ignored).Found()) return false;
- }
- return true;
- }
-
- private:
- friend void BuildTrie<Quant, Bhiksha>(SortedFiles &files, std::vector<uint64_t> &counts, const Config &config, TrieSearch<Quant, Bhiksha> &out, Quant &quant, SortedVocabulary &vocab, BinaryFormat &backing);
-
- // Middles are managed manually so we can delay construction and they don't have to be copyable.
- void FreeMiddles() {
- for (const Middle *i = middle_begin_; i != middle_end_; ++i) {
- i->~Middle();
- }
- std::free(middle_begin_);
- }
-
- typedef trie::BitPackedMiddle<Bhiksha> Middle;
-
- typedef trie::BitPackedLongest Longest;
- Longest longest_;
-
- Middle *middle_begin_, *middle_end_;
- Quant quant_;
-
- typedef ::lm::ngram::trie::Unigram Unigram;
- Unigram unigram_;
-};
-
-} // namespace trie
-} // namespace ngram
-} // namespace lm
-
-#endif // LM_SEARCH_TRIE_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/sizes.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/sizes.cc b/ext/kenlm/lm/sizes.cc
deleted file mode 100644
index dd831c5..0000000
--- a/ext/kenlm/lm/sizes.cc
+++ /dev/null
@@ -1,63 +0,0 @@
-#include "lm/sizes.hh"
-#include "lm/model.hh"
-#include "util/file_piece.hh"
-
-#include <vector>
-#include <iomanip>
-
-namespace lm {
-namespace ngram {
-
-void ShowSizes(const std::vector<uint64_t> &counts, const lm::ngram::Config &config) {
- uint64_t sizes[6];
- sizes[0] = ProbingModel::Size(counts, config);
- sizes[1] = RestProbingModel::Size(counts, config);
- sizes[2] = TrieModel::Size(counts, config);
- sizes[3] = QuantTrieModel::Size(counts, config);
- sizes[4] = ArrayTrieModel::Size(counts, config);
- sizes[5] = QuantArrayTrieModel::Size(counts, config);
- uint64_t max_length = *std::max_element(sizes, sizes + sizeof(sizes) / sizeof(uint64_t));
- uint64_t min_length = *std::min_element(sizes, sizes + sizeof(sizes) / sizeof(uint64_t));
- uint64_t divide;
- char prefix;
- if (min_length < (1 << 10) * 10) {
- prefix = ' ';
- divide = 1;
- } else if (min_length < (1 << 20) * 10) {
- prefix = 'k';
- divide = 1 << 10;
- } else if (min_length < (1ULL << 30) * 10) {
- prefix = 'M';
- divide = 1 << 20;
- } else {
- prefix = 'G';
- divide = 1 << 30;
- }
- long int length = std::max<long int>(2, static_cast<long int>(ceil(log10((double) max_length / divide))));
- std::cerr << "Memory estimate for binary LM:\ntype ";
-
- // right align bytes.
- for (long int i = 0; i < length - 2; ++i) std::cerr << ' ';
-
- std::cerr << prefix << "B\n"
- "probing " << std::setw(length) << (sizes[0] / divide) << " assuming -p " << config.probing_multiplier << "\n"
- "probing " << std::setw(length) << (sizes[1] / divide) << " assuming -r models -p " << config.probing_multiplier << "\n"
- "trie " << std::setw(length) << (sizes[2] / divide) << " without quantization\n"
- "trie " << std::setw(length) << (sizes[3] / divide) << " assuming -q " << (unsigned)config.prob_bits << " -b " << (unsigned)config.backoff_bits << " quantization \n"
- "trie " << std::setw(length) << (sizes[4] / divide) << " assuming -a " << (unsigned)config.pointer_bhiksha_bits << " array pointer compression\n"
- "trie " << std::setw(length) << (sizes[5] / divide) << " assuming -a " << (unsigned)config.pointer_bhiksha_bits << " -q " << (unsigned)config.prob_bits << " -b " << (unsigned)config.backoff_bits<< " array pointer compression and quantization\n";
-}
-
-void ShowSizes(const std::vector<uint64_t> &counts) {
- lm::ngram::Config config;
- ShowSizes(counts, config);
-}
-
-void ShowSizes(const char *file, const lm::ngram::Config &config) {
- std::vector<uint64_t> counts;
- util::FilePiece f(file);
- lm::ReadARPACounts(f, counts);
- ShowSizes(counts, config);
-}
-
-}} //namespaces
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/sizes.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/sizes.hh b/ext/kenlm/lm/sizes.hh
deleted file mode 100644
index eb7e99d..0000000
--- a/ext/kenlm/lm/sizes.hh
+++ /dev/null
@@ -1,17 +0,0 @@
-#ifndef LM_SIZES_H
-#define LM_SIZES_H
-
-#include <vector>
-
-#include <stdint.h>
-
-namespace lm { namespace ngram {
-
-struct Config;
-
-void ShowSizes(const std::vector<uint64_t> &counts, const lm::ngram::Config &config);
-void ShowSizes(const std::vector<uint64_t> &counts);
-void ShowSizes(const char *file, const lm::ngram::Config &config);
-
-}} // namespaces
-#endif // LM_SIZES_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/state.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/state.hh b/ext/kenlm/lm/state.hh
deleted file mode 100644
index 2195dee..0000000
--- a/ext/kenlm/lm/state.hh
+++ /dev/null
@@ -1,125 +0,0 @@
-#ifndef LM_STATE_H
-#define LM_STATE_H
-
-#include "lm/max_order.hh"
-#include "lm/word_index.hh"
-#include "util/murmur_hash.hh"
-
-#include <cstring>
-
-namespace lm {
-namespace ngram {
-
-// This is a POD but if you want memcmp to return the same as operator==, call
-// ZeroRemaining first.
-class State {
- public:
- bool operator==(const State &other) const {
- if (length != other.length) return false;
- return !memcmp(words, other.words, length * sizeof(WordIndex));
- }
-
- // Three way comparison function.
- int Compare(const State &other) const {
- if (length != other.length) return length < other.length ? -1 : 1;
- return memcmp(words, other.words, length * sizeof(WordIndex));
- }
-
- bool operator<(const State &other) const {
- if (length != other.length) return length < other.length;
- return memcmp(words, other.words, length * sizeof(WordIndex)) < 0;
- }
-
- // Call this before using raw memcmp.
- void ZeroRemaining() {
- for (unsigned char i = length; i < KENLM_MAX_ORDER - 1; ++i) {
- words[i] = 0;
- backoff[i] = 0.0;
- }
- }
-
- unsigned char Length() const { return length; }
-
- // You shouldn't need to touch anything below this line, but the members are public so FullState will qualify as a POD.
- // This order minimizes total size of the struct if WordIndex is 64 bit, float is 32 bit, and alignment of 64 bit integers is 64 bit.
- WordIndex words[KENLM_MAX_ORDER - 1];
- float backoff[KENLM_MAX_ORDER - 1];
- unsigned char length;
-};
-
-typedef State Right;
-
-inline uint64_t hash_value(const State &state, uint64_t seed = 0) {
- return util::MurmurHashNative(state.words, sizeof(WordIndex) * state.length, seed);
-}
-
-struct Left {
- bool operator==(const Left &other) const {
- return
- length == other.length &&
- (!length || (pointers[length - 1] == other.pointers[length - 1] && full == other.full));
- }
-
- int Compare(const Left &other) const {
- if (length < other.length) return -1;
- if (length > other.length) return 1;
- if (length == 0) return 0; // Must be full.
- if (pointers[length - 1] > other.pointers[length - 1]) return 1;
- if (pointers[length - 1] < other.pointers[length - 1]) return -1;
- return (int)full - (int)other.full;
- }
-
- bool operator<(const Left &other) const {
- return Compare(other) == -1;
- }
-
- void ZeroRemaining() {
- for (uint64_t * i = pointers + length; i < pointers + KENLM_MAX_ORDER - 1; ++i)
- *i = 0;
- }
-
- uint64_t pointers[KENLM_MAX_ORDER - 1];
- unsigned char length;
- bool full;
-};
-
-inline uint64_t hash_value(const Left &left) {
- unsigned char add[2];
- add[0] = left.length;
- add[1] = left.full;
- return util::MurmurHashNative(add, 2, left.length ? left.pointers[left.length - 1] : 0);
-}
-
-struct ChartState {
- bool operator==(const ChartState &other) const {
- return (right == other.right) && (left == other.left);
- }
-
- int Compare(const ChartState &other) const {
- int lres = left.Compare(other.left);
- if (lres) return lres;
- return right.Compare(other.right);
- }
-
- bool operator<(const ChartState &other) const {
- return Compare(other) < 0;
- }
-
- void ZeroRemaining() {
- left.ZeroRemaining();
- right.ZeroRemaining();
- }
-
- Left left;
- State right;
-};
-
-inline uint64_t hash_value(const ChartState &state) {
- return hash_value(state.right, hash_value(state.left));
-}
-
-
-} // namespace ngram
-} // namespace lm
-
-#endif // LM_STATE_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/test.arpa
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/test.arpa b/ext/kenlm/lm/test.arpa
deleted file mode 100644
index c4d2e6d..0000000
--- a/ext/kenlm/lm/test.arpa
+++ /dev/null
@@ -1,124 +0,0 @@
-
-\data\
-ngram 1=37
-ngram 2=47
-ngram 3=11
-ngram 4=6
-ngram 5=4
-
-\1-grams:
--1.383514 , -0.30103
--1.139057 . -0.845098
--1.029493 </s>
--99 <s> -0.4149733
--1.995635 <unk> -20
--1.285941 a -0.69897
--1.687872 also -0.30103
--1.687872 beyond -0.30103
--1.687872 biarritz -0.30103
--1.687872 call -0.30103
--1.687872 concerns -0.30103
--1.687872 consider -0.30103
--1.687872 considering -0.30103
--1.687872 for -0.30103
--1.509559 higher -0.30103
--1.687872 however -0.30103
--1.687872 i -0.30103
--1.687872 immediate -0.30103
--1.687872 in -0.30103
--1.687872 is -0.30103
--1.285941 little -0.69897
--1.383514 loin -0.30103
--1.687872 look -0.30103
--1.285941 looking -0.4771212
--1.206319 more -0.544068
--1.509559 on -0.4771212
--1.509559 screening -0.4771212
--1.687872 small -0.30103
--1.687872 the -0.30103
--1.687872 to -0.30103
--1.687872 watch -0.30103
--1.687872 watching -0.30103
--1.687872 what -0.30103
--1.687872 would -0.30103
--3.141592 foo
--2.718281 bar 3.0
--6.535897 baz -0.0
-
-\2-grams:
--0.6925742 , .
--0.7522095 , however
--0.7522095 , is
--0.0602359 . </s>
--0.4846522 <s> looking -0.4771214
--1.051485 <s> screening
--1.07153 <s> the
--1.07153 <s> watching
--1.07153 <s> what
--0.09132547 a little -0.69897
--0.2922095 also call
--0.2922095 beyond immediate
--0.2705918 biarritz .
--0.2922095 call for
--0.2922095 concerns in
--0.2922095 consider watch
--0.2922095 considering consider
--0.2834328 for ,
--0.5511513 higher more
--0.5845945 higher small
--0.2834328 however ,
--0.2922095 i would
--0.2922095 immediate concerns
--0.2922095 in biarritz
--0.2922095 is to
--0.09021038 little more -0.1998621
--0.7273645 loin ,
--0.6925742 loin .
--0.6708385 loin </s>
--0.2922095 look beyond
--0.4638903 looking higher
--0.4638903 looking on -0.4771212
--0.5136299 more . -0.4771212
--0.3561665 more loin
--0.1649931 on a -0.4771213
--0.1649931 screening a -0.4771213
--0.2705918 small .
--0.287799 the screening
--0.2922095 to look
--0.2622373 watch </s>
--0.2922095 watching considering
--0.2922095 what i
--0.2922095 would also
--2 also would -6
--15 <unk> <unk> -2
--4 <unk> however -1
--6 foo bar
-
-\3-grams:
--0.01916512 more . </s>
--0.0283603 on a little -0.4771212
--0.0283603 screening a little -0.4771212
--0.01660496 a little more -0.09409451
--0.3488368 <s> looking higher
--0.3488368 <s> looking on -0.4771212
--0.1892331 little more loin
--0.04835128 looking on a -0.4771212
--3 also would consider -7
--6 <unk> however <unk> -12
--7 to look a
-
-\4-grams:
--0.009249173 looking on a little -0.4771212
--0.005464747 on a little more -0.4771212
--0.005464747 screening a little more
--0.1453306 a little more loin
--0.01552657 <s> looking on a -0.4771212
--4 also would consider higher -8
-
-\5-grams:
--0.003061223 <s> looking on a little
--0.001813953 looking on a little more
--0.0432557 on a little more loin
--5 also would consider higher looking
-
-\end\
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/test_nounk.arpa
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/test_nounk.arpa b/ext/kenlm/lm/test_nounk.arpa
deleted file mode 100644
index e38fc85..0000000
--- a/ext/kenlm/lm/test_nounk.arpa
+++ /dev/null
@@ -1,120 +0,0 @@
-
-\data\
-ngram 1=36
-ngram 2=45
-ngram 3=10
-ngram 4=6
-ngram 5=4
-
-\1-grams:
--1.383514 , -0.30103
--1.139057 . -0.845098
--1.029493 </s>
--99 <s> -0.4149733
--1.285941 a -0.69897
--1.687872 also -0.30103
--1.687872 beyond -0.30103
--1.687872 biarritz -0.30103
--1.687872 call -0.30103
--1.687872 concerns -0.30103
--1.687872 consider -0.30103
--1.687872 considering -0.30103
--1.687872 for -0.30103
--1.509559 higher -0.30103
--1.687872 however -0.30103
--1.687872 i -0.30103
--1.687872 immediate -0.30103
--1.687872 in -0.30103
--1.687872 is -0.30103
--1.285941 little -0.69897
--1.383514 loin -0.30103
--1.687872 look -0.30103
--1.285941 looking -0.4771212
--1.206319 more -0.544068
--1.509559 on -0.4771212
--1.509559 screening -0.4771212
--1.687872 small -0.30103
--1.687872 the -0.30103
--1.687872 to -0.30103
--1.687872 watch -0.30103
--1.687872 watching -0.30103
--1.687872 what -0.30103
--1.687872 would -0.30103
--3.141592 foo
--2.718281 bar 3.0
--6.535897 baz -0.0
-
-\2-grams:
--0.6925742 , .
--0.7522095 , however
--0.7522095 , is
--0.0602359 . </s>
--0.4846522 <s> looking -0.4771214
--1.051485 <s> screening
--1.07153 <s> the
--1.07153 <s> watching
--1.07153 <s> what
--0.09132547 a little -0.69897
--0.2922095 also call
--0.2922095 beyond immediate
--0.2705918 biarritz .
--0.2922095 call for
--0.2922095 concerns in
--0.2922095 consider watch
--0.2922095 considering consider
--0.2834328 for ,
--0.5511513 higher more
--0.5845945 higher small
--0.2834328 however ,
--0.2922095 i would
--0.2922095 immediate concerns
--0.2922095 in biarritz
--0.2922095 is to
--0.09021038 little more -0.1998621
--0.7273645 loin ,
--0.6925742 loin .
--0.6708385 loin </s>
--0.2922095 look beyond
--0.4638903 looking higher
--0.4638903 looking on -0.4771212
--0.5136299 more . -0.4771212
--0.3561665 more loin
--0.1649931 on a -0.4771213
--0.1649931 screening a -0.4771213
--0.2705918 small .
--0.287799 the screening
--0.2922095 to look
--0.2622373 watch </s>
--0.2922095 watching considering
--0.2922095 what i
--0.2922095 would also
--2 also would -6
--6 foo bar
-
-\3-grams:
--0.01916512 more . </s>
--0.0283603 on a little -0.4771212
--0.0283603 screening a little -0.4771212
--0.01660496 a little more -0.09409451
--0.3488368 <s> looking higher
--0.3488368 <s> looking on -0.4771212
--0.1892331 little more loin
--0.04835128 looking on a -0.4771212
--3 also would consider -7
--7 to look a
-
-\4-grams:
--0.009249173 looking on a little -0.4771212
--0.005464747 on a little more -0.4771212
--0.005464747 screening a little more
--0.1453306 a little more loin
--0.01552657 <s> looking on a -0.4771212
--4 also would consider higher -8
-
-\5-grams:
--0.003061223 <s> looking on a little
--0.001813953 looking on a little more
--0.0432557 on a little more loin
--5 also would consider higher looking
-
-\end\
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/trie.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/trie.cc b/ext/kenlm/lm/trie.cc
deleted file mode 100644
index 72ad544..0000000
--- a/ext/kenlm/lm/trie.cc
+++ /dev/null
@@ -1,131 +0,0 @@
-#include "lm/trie.hh"
-
-#include "lm/bhiksha.hh"
-#include "util/bit_packing.hh"
-#include "util/exception.hh"
-#include "util/sorted_uniform.hh"
-
-#include <cassert>
-
-namespace lm {
-namespace ngram {
-namespace trie {
-namespace {
-
-class KeyAccessor {
- public:
- KeyAccessor(const void *base, uint64_t key_mask, uint8_t key_bits, uint8_t total_bits)
- : base_(reinterpret_cast<const uint8_t*>(base)), key_mask_(key_mask), key_bits_(key_bits), total_bits_(total_bits) {}
-
- typedef uint64_t Key;
-
- Key operator()(uint64_t index) const {
- return util::ReadInt57(base_, index * static_cast<uint64_t>(total_bits_), key_bits_, key_mask_);
- }
-
- private:
- const uint8_t *const base_;
- const WordIndex key_mask_;
- const uint8_t key_bits_, total_bits_;
-};
-
-bool FindBitPacked(const void *base, uint64_t key_mask, uint8_t key_bits, uint8_t total_bits, uint64_t begin_index, uint64_t end_index, const uint64_t max_vocab, const uint64_t key, uint64_t &at_index) {
- KeyAccessor accessor(base, key_mask, key_bits, total_bits);
- if (!util::BoundedSortedUniformFind<uint64_t, KeyAccessor, util::PivotSelect<sizeof(WordIndex)>::T>(accessor, begin_index - 1, (uint64_t)0, end_index, max_vocab, key, at_index)) return false;
- return true;
-}
-} // namespace
-
-uint64_t BitPacked::BaseSize(uint64_t entries, uint64_t max_vocab, uint8_t remaining_bits) {
- uint8_t total_bits = util::RequiredBits(max_vocab) + remaining_bits;
- // Extra entry for next pointer at the end.
- // +7 then / 8 to round up bits and convert to bytes
- // +sizeof(uint64_t) so that ReadInt57 etc don't go segfault.
- // Note that this waste is O(order), not O(number of ngrams).
- return ((1 + entries) * total_bits + 7) / 8 + sizeof(uint64_t);
-}
-
-void BitPacked::BaseInit(void *base, uint64_t max_vocab, uint8_t remaining_bits) {
- util::BitPackingSanity();
- word_bits_ = util::RequiredBits(max_vocab);
- word_mask_ = (1ULL << word_bits_) - 1ULL;
- if (word_bits_ > 57) UTIL_THROW(util::Exception, "Sorry, word indices more than " << (1ULL << 57) << " are not implemented. Edit util/bit_packing.hh and fix the bit packing functions.");
- total_bits_ = word_bits_ + remaining_bits;
-
- base_ = static_cast<uint8_t*>(base);
- insert_index_ = 0;
- max_vocab_ = max_vocab;
-}
-
-template <class Bhiksha> uint64_t BitPackedMiddle<Bhiksha>::Size(uint8_t quant_bits, uint64_t entries, uint64_t max_vocab, uint64_t max_ptr, const Config &config) {
- return Bhiksha::Size(entries + 1, max_ptr, config) + BaseSize(entries, max_vocab, quant_bits + Bhiksha::InlineBits(entries + 1, max_ptr, config));
-}
-
-template <class Bhiksha> BitPackedMiddle<Bhiksha>::BitPackedMiddle(void *base, uint8_t quant_bits, uint64_t entries, uint64_t max_vocab, uint64_t max_next, const BitPacked &next_source, const Config &config) :
- BitPacked(),
- quant_bits_(quant_bits),
- // If the offset of the method changes, also change TrieSearch::UpdateConfigFromBinary.
- bhiksha_(base, entries + 1, max_next, config),
- next_source_(&next_source) {
- if (entries + 1 >= (1ULL << 57) || (max_next >= (1ULL << 57))) UTIL_THROW(util::Exception, "Sorry, this does not support more than " << (1ULL << 57) << " n-grams of a particular order. Edit util/bit_packing.hh and fix the bit packing functions.");
- BaseInit(reinterpret_cast<uint8_t*>(base) + Bhiksha::Size(entries + 1, max_next, config), max_vocab, quant_bits_ + bhiksha_.InlineBits());
-}
-
-template <class Bhiksha> util::BitAddress BitPackedMiddle<Bhiksha>::Insert(WordIndex word) {
- assert(word <= word_mask_);
- uint64_t at_pointer = insert_index_ * total_bits_;
-
- util::WriteInt57(base_, at_pointer, word_bits_, word);
- at_pointer += word_bits_;
- util::BitAddress ret(base_, at_pointer);
- at_pointer += quant_bits_;
- uint64_t next = next_source_->InsertIndex();
- bhiksha_.WriteNext(base_, at_pointer, insert_index_, next);
- ++insert_index_;
- return ret;
-}
-
-template <class Bhiksha> util::BitAddress BitPackedMiddle<Bhiksha>::Find(WordIndex word, NodeRange &range, uint64_t &pointer) const {
- uint64_t at_pointer;
- if (!FindBitPacked(base_, word_mask_, word_bits_, total_bits_, range.begin, range.end, max_vocab_, word, at_pointer)) {
- return util::BitAddress(NULL, 0);
- }
- pointer = at_pointer;
- at_pointer *= total_bits_;
- at_pointer += word_bits_;
- bhiksha_.ReadNext(base_, at_pointer + quant_bits_, pointer, total_bits_, range);
-
- return util::BitAddress(base_, at_pointer);
-}
-
-template <class Bhiksha> void BitPackedMiddle<Bhiksha>::FinishedLoading(uint64_t next_end, const Config &config) {
- // Write at insert_index. . .
- uint64_t last_next_write = insert_index_ * total_bits_ +
- // at the offset where the next pointers are stored.
- (total_bits_ - bhiksha_.InlineBits());
- bhiksha_.WriteNext(base_, last_next_write, insert_index_, next_end);
- bhiksha_.FinishedLoading(config);
-}
-
-util::BitAddress BitPackedLongest::Insert(WordIndex index) {
- assert(index <= word_mask_);
- uint64_t at_pointer = insert_index_ * total_bits_;
- util::WriteInt57(base_, at_pointer, word_bits_, index);
- at_pointer += word_bits_;
- ++insert_index_;
- return util::BitAddress(base_, at_pointer);
-}
-
-util::BitAddress BitPackedLongest::Find(WordIndex word, const NodeRange &range) const {
- uint64_t at_pointer;
- if (!FindBitPacked(base_, word_mask_, word_bits_, total_bits_, range.begin, range.end, max_vocab_, word, at_pointer)) return util::BitAddress(NULL, 0);
- at_pointer = at_pointer * total_bits_ + word_bits_;
- return util::BitAddress(base_, at_pointer);
-}
-
-template class BitPackedMiddle<DontBhiksha>;
-template class BitPackedMiddle<ArrayBhiksha>;
-
-} // namespace trie
-} // namespace ngram
-} // namespace lm
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/trie.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/trie.hh b/ext/kenlm/lm/trie.hh
deleted file mode 100644
index b7f0458..0000000
--- a/ext/kenlm/lm/trie.hh
+++ /dev/null
@@ -1,146 +0,0 @@
-#ifndef LM_TRIE_H
-#define LM_TRIE_H
-
-#include "lm/weights.hh"
-#include "lm/word_index.hh"
-#include "util/bit_packing.hh"
-
-#include <cstddef>
-
-#include <stdint.h>
-
-namespace lm {
-namespace ngram {
-struct Config;
-namespace trie {
-
-struct NodeRange {
- uint64_t begin, end;
-};
-
-// TODO: if the number of unigrams is a concern, also bit pack these records.
-struct UnigramValue {
- ProbBackoff weights;
- uint64_t next;
- uint64_t Next() const { return next; }
-};
-
-class UnigramPointer {
- public:
- explicit UnigramPointer(const ProbBackoff &to) : to_(&to) {}
-
- UnigramPointer() : to_(NULL) {}
-
- bool Found() const { return to_ != NULL; }
-
- float Prob() const { return to_->prob; }
- float Backoff() const { return to_->backoff; }
- float Rest() const { return Prob(); }
-
- private:
- const ProbBackoff *to_;
-};
-
-class Unigram {
- public:
- Unigram() {}
-
- void Init(void *start) {
- unigram_ = static_cast<UnigramValue*>(start);
- }
-
- static uint64_t Size(uint64_t count) {
- // +1 in case unknown doesn't appear. +1 for the final next.
- return (count + 2) * sizeof(UnigramValue);
- }
-
- const ProbBackoff &Lookup(WordIndex index) const { return unigram_[index].weights; }
-
- ProbBackoff &Unknown() { return unigram_[0].weights; }
-
- UnigramValue *Raw() {
- return unigram_;
- }
-
- UnigramPointer Find(WordIndex word, NodeRange &next) const {
- UnigramValue *val = unigram_ + word;
- next.begin = val->next;
- next.end = (val+1)->next;
- return UnigramPointer(val->weights);
- }
-
- private:
- UnigramValue *unigram_;
-};
-
-class BitPacked {
- public:
- BitPacked() {}
-
- uint64_t InsertIndex() const {
- return insert_index_;
- }
-
- protected:
- static uint64_t BaseSize(uint64_t entries, uint64_t max_vocab, uint8_t remaining_bits);
-
- void BaseInit(void *base, uint64_t max_vocab, uint8_t remaining_bits);
-
- uint8_t word_bits_;
- uint8_t total_bits_;
- uint64_t word_mask_;
-
- uint8_t *base_;
-
- uint64_t insert_index_, max_vocab_;
-};
-
-template <class Bhiksha> class BitPackedMiddle : public BitPacked {
- public:
- static uint64_t Size(uint8_t quant_bits, uint64_t entries, uint64_t max_vocab, uint64_t max_next, const Config &config);
-
- // next_source need not be initialized.
- BitPackedMiddle(void *base, uint8_t quant_bits, uint64_t entries, uint64_t max_vocab, uint64_t max_next, const BitPacked &next_source, const Config &config);
-
- util::BitAddress Insert(WordIndex word);
-
- void FinishedLoading(uint64_t next_end, const Config &config);
-
- util::BitAddress Find(WordIndex word, NodeRange &range, uint64_t &pointer) const;
-
- util::BitAddress ReadEntry(uint64_t pointer, NodeRange &range) {
- uint64_t addr = pointer * total_bits_;
- addr += word_bits_;
- bhiksha_.ReadNext(base_, addr + quant_bits_, pointer, total_bits_, range);
- return util::BitAddress(base_, addr);
- }
-
- private:
- uint8_t quant_bits_;
- Bhiksha bhiksha_;
-
- const BitPacked *next_source_;
-};
-
-class BitPackedLongest : public BitPacked {
- public:
- static uint64_t Size(uint8_t quant_bits, uint64_t entries, uint64_t max_vocab) {
- return BaseSize(entries, max_vocab, quant_bits);
- }
-
- BitPackedLongest() {}
-
- void Init(void *base, uint8_t quant_bits, uint64_t max_vocab) {
- BaseInit(base, max_vocab, quant_bits);
- }
-
- util::BitAddress Insert(WordIndex word);
-
- util::BitAddress Find(WordIndex word, const NodeRange &node) const;
-};
-
-} // namespace trie
-} // namespace ngram
-} // namespace lm
-
-#endif // LM_TRIE_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/trie_sort.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/trie_sort.cc b/ext/kenlm/lm/trie_sort.cc
deleted file mode 100644
index 33a2f96..0000000
--- a/ext/kenlm/lm/trie_sort.cc
+++ /dev/null
@@ -1,304 +0,0 @@
-#include "lm/trie_sort.hh"
-
-#include "lm/config.hh"
-#include "lm/lm_exception.hh"
-#include "lm/read_arpa.hh"
-#include "lm/vocab.hh"
-#include "lm/weights.hh"
-#include "lm/word_index.hh"
-#include "util/file_piece.hh"
-#include "util/mmap.hh"
-#include "util/proxy_iterator.hh"
-#include "util/sized_iterator.hh"
-
-#include <algorithm>
-#include <cstring>
-#include <cstdio>
-#include <cstdlib>
-#include <deque>
-#include <iterator>
-#include <limits>
-#include <vector>
-
-namespace lm {
-namespace ngram {
-namespace trie {
-namespace {
-
-typedef util::SizedIterator NGramIter;
-
-// Proxy for an entry except there is some extra cruft between the entries. This is used to sort (n-1)-grams using the same memory as the sorted n-grams.
-class PartialViewProxy {
- public:
- PartialViewProxy() : attention_size_(0), inner_() {}
-
- PartialViewProxy(void *ptr, std::size_t block_size, std::size_t attention_size) : attention_size_(attention_size), inner_(ptr, block_size) {}
-
- operator std::string() const {
- return std::string(reinterpret_cast<const char*>(inner_.Data()), attention_size_);
- }
-
- PartialViewProxy &operator=(const PartialViewProxy &from) {
- memcpy(inner_.Data(), from.inner_.Data(), attention_size_);
- return *this;
- }
-
- PartialViewProxy &operator=(const std::string &from) {
- memcpy(inner_.Data(), from.data(), attention_size_);
- return *this;
- }
-
- const void *Data() const { return inner_.Data(); }
- void *Data() { return inner_.Data(); }
-
- friend void swap(PartialViewProxy first, PartialViewProxy second) {
- std::swap_ranges(reinterpret_cast<char*>(first.Data()), reinterpret_cast<char*>(first.Data()) + first.attention_size_, reinterpret_cast<char*>(second.Data()));
- }
-
- private:
- friend class util::ProxyIterator<PartialViewProxy>;
-
- typedef std::string value_type;
-
- const std::size_t attention_size_;
-
- typedef util::SizedInnerIterator InnerIterator;
- InnerIterator &Inner() { return inner_; }
- const InnerIterator &Inner() const { return inner_; }
- InnerIterator inner_;
-};
-
-typedef util::ProxyIterator<PartialViewProxy> PartialIter;
-
-FILE *DiskFlush(const void *mem_begin, const void *mem_end, const std::string &temp_prefix) {
- util::scoped_fd file(util::MakeTemp(temp_prefix));
- util::WriteOrThrow(file.get(), mem_begin, (uint8_t*)mem_end - (uint8_t*)mem_begin);
- return util::FDOpenOrThrow(file);
-}
-
-FILE *WriteContextFile(uint8_t *begin, uint8_t *end, const std::string &temp_prefix, std::size_t entry_size, unsigned char order) {
- const size_t context_size = sizeof(WordIndex) * (order - 1);
- // Sort just the contexts using the same memory.
- PartialIter context_begin(PartialViewProxy(begin + sizeof(WordIndex), entry_size, context_size));
- PartialIter context_end(PartialViewProxy(end + sizeof(WordIndex), entry_size, context_size));
-
-#if defined(_WIN32) || defined(_WIN64)
- std::stable_sort
-#else
- std::sort
-#endif
- (context_begin, context_end, util::SizedCompare<EntryCompare, PartialViewProxy>(EntryCompare(order - 1)));
-
- util::scoped_FILE out(util::FMakeTemp(temp_prefix));
-
- // Write out to file and uniqueify at the same time. Could have used unique_copy if there was an appropriate OutputIterator.
- if (context_begin == context_end) return out.release();
- PartialIter i(context_begin);
- util::WriteOrThrow(out.get(), i->Data(), context_size);
- const void *previous = i->Data();
- ++i;
- for (; i != context_end; ++i) {
- if (memcmp(previous, i->Data(), context_size)) {
- util::WriteOrThrow(out.get(), i->Data(), context_size);
- previous = i->Data();
- }
- }
- return out.release();
-}
-
-struct ThrowCombine {
- void operator()(std::size_t entry_size, unsigned char order, const void *first, const void *second, FILE * /*out*/) const {
- const WordIndex *base = reinterpret_cast<const WordIndex*>(first);
- FormatLoadException e;
- e << "Duplicate n-gram detected with vocab ids";
- for (const WordIndex *i = base; i != base + order; ++i) {
- e << ' ' << *i;
- }
- throw e;
- }
-};
-
-// Useful for context files that just contain records with no value.
-struct FirstCombine {
- void operator()(std::size_t entry_size, unsigned char /*order*/, const void *first, const void * /*second*/, FILE *out) const {
- util::WriteOrThrow(out, first, entry_size);
- }
-};
-
-template <class Combine> FILE *MergeSortedFiles(FILE *first_file, FILE *second_file, const std::string &temp_prefix, std::size_t weights_size, unsigned char order, const Combine &combine) {
- std::size_t entry_size = sizeof(WordIndex) * order + weights_size;
- RecordReader first, second;
- first.Init(first_file, entry_size);
- second.Init(second_file, entry_size);
- util::scoped_FILE out_file(util::FMakeTemp(temp_prefix));
- EntryCompare less(order);
- while (first && second) {
- if (less(first.Data(), second.Data())) {
- util::WriteOrThrow(out_file.get(), first.Data(), entry_size);
- ++first;
- } else if (less(second.Data(), first.Data())) {
- util::WriteOrThrow(out_file.get(), second.Data(), entry_size);
- ++second;
- } else {
- combine(entry_size, order, first.Data(), second.Data(), out_file.get());
- ++first; ++second;
- }
- }
- for (RecordReader &remains = (first ? first : second); remains; ++remains) {
- util::WriteOrThrow(out_file.get(), remains.Data(), entry_size);
- }
- return out_file.release();
-}
-
-} // namespace
-
-void RecordReader::Init(FILE *file, std::size_t entry_size) {
- entry_size_ = entry_size;
- data_.reset(malloc(entry_size));
- UTIL_THROW_IF(!data_.get(), util::ErrnoException, "Failed to malloc read buffer");
- file_ = file;
- if (file) {
- rewind(file);
- remains_ = true;
- ++*this;
- } else {
- remains_ = false;
- }
-}
-
-void RecordReader::Overwrite(const void *start, std::size_t amount) {
- long internal = (uint8_t*)start - (uint8_t*)data_.get();
- UTIL_THROW_IF(fseek(file_, internal - entry_size_, SEEK_CUR), util::ErrnoException, "Couldn't seek backwards for revision");
- util::WriteOrThrow(file_, start, amount);
- long forward = entry_size_ - internal - amount;
-#if !defined(_WIN32) && !defined(_WIN64)
- if (forward)
-#endif
- UTIL_THROW_IF(fseek(file_, forward, SEEK_CUR), util::ErrnoException, "Couldn't seek forwards past revision");
-}
-
-void RecordReader::Rewind() {
- if (file_) {
- rewind(file_);
- remains_ = true;
- ++*this;
- } else {
- remains_ = false;
- }
-}
-
-SortedFiles::SortedFiles(const Config &config, util::FilePiece &f, std::vector<uint64_t> &counts, size_t buffer, const std::string &file_prefix, SortedVocabulary &vocab) {
- PositiveProbWarn warn(config.positive_log_probability);
- unigram_.reset(util::MakeTemp(file_prefix));
- {
- // In case <unk> appears.
- size_t size_out = (counts[0] + 1) * sizeof(ProbBackoff);
- util::scoped_mmap unigram_mmap(util::MapZeroedWrite(unigram_.get(), size_out), size_out);
- Read1Grams(f, counts[0], vocab, reinterpret_cast<ProbBackoff*>(unigram_mmap.get()), warn);
- CheckSpecials(config, vocab);
- if (!vocab.SawUnk()) ++counts[0];
- }
-
- // Only use as much buffer as we need.
- size_t buffer_use = 0;
- for (unsigned int order = 2; order < counts.size(); ++order) {
- buffer_use = std::max<size_t>(buffer_use, static_cast<size_t>((sizeof(WordIndex) * order + 2 * sizeof(float)) * counts[order - 1]));
- }
- buffer_use = std::max<size_t>(buffer_use, static_cast<size_t>((sizeof(WordIndex) * counts.size() + sizeof(float)) * counts.back()));
- buffer = std::min<size_t>(buffer, buffer_use);
-
- util::scoped_malloc mem;
- mem.reset(malloc(buffer));
- if (!mem.get()) UTIL_THROW(util::ErrnoException, "malloc failed for sort buffer size " << buffer);
-
- for (unsigned char order = 2; order <= counts.size(); ++order) {
- ConvertToSorted(f, vocab, counts, file_prefix, order, warn, mem.get(), buffer);
- }
- ReadEnd(f);
-}
-
-namespace {
-class Closer {
- public:
- explicit Closer(std::deque<FILE*> &files) : files_(files) {}
-
- ~Closer() {
- for (std::deque<FILE*>::iterator i = files_.begin(); i != files_.end(); ++i) {
- util::scoped_FILE deleter(*i);
- }
- }
-
- void PopFront() {
- util::scoped_FILE deleter(files_.front());
- files_.pop_front();
- }
- private:
- std::deque<FILE*> &files_;
-};
-} // namespace
-
-void SortedFiles::ConvertToSorted(util::FilePiece &f, const SortedVocabulary &vocab, const std::vector<uint64_t> &counts, const std::string &file_prefix, unsigned char order, PositiveProbWarn &warn, void *mem, std::size_t mem_size) {
- ReadNGramHeader(f, order);
- const size_t count = counts[order - 1];
- // Size of weights. Does it include backoff?
- const size_t words_size = sizeof(WordIndex) * order;
- const size_t weights_size = sizeof(float) + ((order == counts.size()) ? 0 : sizeof(float));
- const size_t entry_size = words_size + weights_size;
- const size_t batch_size = std::min(count, mem_size / entry_size);
- uint8_t *const begin = reinterpret_cast<uint8_t*>(mem);
-
- std::deque<FILE*> files, contexts;
- Closer files_closer(files), contexts_closer(contexts);
-
- for (std::size_t batch = 0, done = 0; done < count; ++batch) {
- uint8_t *out = begin;
- uint8_t *out_end = out + std::min(count - done, batch_size) * entry_size;
- if (order == counts.size()) {
- for (; out != out_end; out += entry_size) {
- std::reverse_iterator<WordIndex*> it(reinterpret_cast<WordIndex*>(out) + order);
- ReadNGram(f, order, vocab, it, *reinterpret_cast<Prob*>(out + words_size), warn);
- }
- } else {
- for (; out != out_end; out += entry_size) {
- std::reverse_iterator<WordIndex*> it(reinterpret_cast<WordIndex*>(out) + order);
- ReadNGram(f, order, vocab, it, *reinterpret_cast<ProbBackoff*>(out + words_size), warn);
- }
- }
- // Sort full records by full n-gram.
- util::SizedProxy proxy_begin(begin, entry_size), proxy_end(out_end, entry_size);
- // parallel_sort uses too much RAM. TODO: figure out why windows sort doesn't like my proxies.
-#if defined(_WIN32) || defined(_WIN64)
- std::stable_sort
-#else
- std::sort
-#endif
- (NGramIter(proxy_begin), NGramIter(proxy_end), util::SizedCompare<EntryCompare>(EntryCompare(order)));
- files.push_back(DiskFlush(begin, out_end, file_prefix));
- contexts.push_back(WriteContextFile(begin, out_end, file_prefix, entry_size, order));
-
- done += (out_end - begin) / entry_size;
- }
-
- // All individual files created. Merge them.
-
- while (files.size() > 1) {
- files.push_back(MergeSortedFiles(files[0], files[1], file_prefix, weights_size, order, ThrowCombine()));
- files_closer.PopFront();
- files_closer.PopFront();
- contexts.push_back(MergeSortedFiles(contexts[0], contexts[1], file_prefix, 0, order - 1, FirstCombine()));
- contexts_closer.PopFront();
- contexts_closer.PopFront();
- }
-
- if (!files.empty()) {
- // Steal from closers.
- full_[order - 2].reset(files.front());
- files.pop_front();
- context_[order - 2].reset(contexts.front());
- contexts.pop_front();
- }
-}
-
-} // namespace trie
-} // namespace ngram
-} // namespace lm
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/trie_sort.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/trie_sort.hh b/ext/kenlm/lm/trie_sort.hh
deleted file mode 100644
index 594efee..0000000
--- a/ext/kenlm/lm/trie_sort.hh
+++ /dev/null
@@ -1,114 +0,0 @@
-// Step of trie builder: create sorted files.
-
-#ifndef LM_TRIE_SORT_H
-#define LM_TRIE_SORT_H
-
-#include "lm/max_order.hh"
-#include "lm/word_index.hh"
-
-#include "util/file.hh"
-#include "util/scoped.hh"
-
-#include <cstddef>
-#include <functional>
-#include <string>
-#include <vector>
-
-#include <stdint.h>
-
-namespace util {
-class FilePiece;
-} // namespace util
-
-namespace lm {
-class PositiveProbWarn;
-namespace ngram {
-class SortedVocabulary;
-struct Config;
-
-namespace trie {
-
-class EntryCompare : public std::binary_function<const void*, const void*, bool> {
- public:
- explicit EntryCompare(unsigned char order) : order_(order) {}
-
- bool operator()(const void *first_void, const void *second_void) const {
- const WordIndex *first = static_cast<const WordIndex*>(first_void);
- const WordIndex *second = static_cast<const WordIndex*>(second_void);
- const WordIndex *end = first + order_;
- for (; first != end; ++first, ++second) {
- if (*first < *second) return true;
- if (*first > *second) return false;
- }
- return false;
- }
- private:
- unsigned char order_;
-};
-
-class RecordReader {
- public:
- RecordReader() : remains_(true) {}
-
- void Init(FILE *file, std::size_t entry_size);
-
- void *Data() { return data_.get(); }
- const void *Data() const { return data_.get(); }
-
- RecordReader &operator++() {
- std::size_t ret = fread(data_.get(), entry_size_, 1, file_);
- if (!ret) {
- UTIL_THROW_IF(!feof(file_), util::ErrnoException, "Error reading temporary file");
- remains_ = false;
- }
- return *this;
- }
-
- operator bool() const { return remains_; }
-
- void Rewind();
-
- std::size_t EntrySize() const { return entry_size_; }
-
- void Overwrite(const void *start, std::size_t amount);
-
- private:
- FILE *file_;
-
- util::scoped_malloc data_;
-
- bool remains_;
-
- std::size_t entry_size_;
-};
-
-class SortedFiles {
- public:
- // Build from ARPA
- SortedFiles(const Config &config, util::FilePiece &f, std::vector<uint64_t> &counts, std::size_t buffer, const std::string &file_prefix, SortedVocabulary &vocab);
-
- int StealUnigram() {
- return unigram_.release();
- }
-
- FILE *Full(unsigned char order) {
- return full_[order - 2].get();
- }
-
- FILE *Context(unsigned char of_order) {
- return context_[of_order - 2].get();
- }
-
- private:
- void ConvertToSorted(util::FilePiece &f, const SortedVocabulary &vocab, const std::vector<uint64_t> &counts, const std::string &prefix, unsigned char order, PositiveProbWarn &warn, void *mem, std::size_t mem_size);
-
- util::scoped_fd unigram_;
-
- util::scoped_FILE full_[KENLM_MAX_ORDER - 1], context_[KENLM_MAX_ORDER - 1];
-};
-
-} // namespace trie
-} // namespace ngram
-} // namespace lm
-
-#endif // LM_TRIE_SORT_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/value.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/value.hh b/ext/kenlm/lm/value.hh
deleted file mode 100644
index d2425cc..0000000
--- a/ext/kenlm/lm/value.hh
+++ /dev/null
@@ -1,158 +0,0 @@
-#ifndef LM_VALUE_H
-#define LM_VALUE_H
-
-#include "lm/config.hh"
-#include "lm/model_type.hh"
-#include "lm/value_build.hh"
-#include "lm/weights.hh"
-#include "util/bit_packing.hh"
-
-#include <stdint.h>
-
-namespace lm {
-namespace ngram {
-
-// Template proxy for probing unigrams and middle.
-template <class Weights> class GenericProbingProxy {
- public:
- explicit GenericProbingProxy(const Weights &to) : to_(&to) {}
-
- GenericProbingProxy() : to_(0) {}
-
- bool Found() const { return to_ != 0; }
-
- float Prob() const {
- util::FloatEnc enc;
- enc.f = to_->prob;
- enc.i |= util::kSignBit;
- return enc.f;
- }
-
- float Backoff() const { return to_->backoff; }
-
- bool IndependentLeft() const {
- util::FloatEnc enc;
- enc.f = to_->prob;
- return enc.i & util::kSignBit;
- }
-
- protected:
- const Weights *to_;
-};
-
-// Basic proxy for trie unigrams.
-template <class Weights> class GenericTrieUnigramProxy {
- public:
- explicit GenericTrieUnigramProxy(const Weights &to) : to_(&to) {}
-
- GenericTrieUnigramProxy() : to_(0) {}
-
- bool Found() const { return to_ != 0; }
- float Prob() const { return to_->prob; }
- float Backoff() const { return to_->backoff; }
- float Rest() const { return Prob(); }
-
- protected:
- const Weights *to_;
-};
-
-struct BackoffValue {
- typedef ProbBackoff Weights;
- static const ModelType kProbingModelType = PROBING;
-
- class ProbingProxy : public GenericProbingProxy<Weights> {
- public:
- explicit ProbingProxy(const Weights &to) : GenericProbingProxy<Weights>(to) {}
- ProbingProxy() {}
- float Rest() const { return Prob(); }
- };
-
- class TrieUnigramProxy : public GenericTrieUnigramProxy<Weights> {
- public:
- explicit TrieUnigramProxy(const Weights &to) : GenericTrieUnigramProxy<Weights>(to) {}
- TrieUnigramProxy() {}
- float Rest() const { return Prob(); }
- };
-
- struct ProbingEntry {
- typedef uint64_t Key;
- typedef Weights Value;
- uint64_t key;
- ProbBackoff value;
- uint64_t GetKey() const { return key; }
- };
-
- struct TrieUnigramValue {
- Weights weights;
- uint64_t next;
- uint64_t Next() const { return next; }
- };
-
- const static bool kDifferentRest = false;
-
- template <class Model, class C> void Callback(const Config &, unsigned int, typename Model::Vocabulary &, C &callback) {
- NoRestBuild build;
- callback(build);
- }
-};
-
-struct RestValue {
- typedef RestWeights Weights;
- static const ModelType kProbingModelType = REST_PROBING;
-
- class ProbingProxy : public GenericProbingProxy<RestWeights> {
- public:
- explicit ProbingProxy(const Weights &to) : GenericProbingProxy<RestWeights>(to) {}
- ProbingProxy() {}
- float Rest() const { return to_->rest; }
- };
-
- class TrieUnigramProxy : public GenericTrieUnigramProxy<Weights> {
- public:
- explicit TrieUnigramProxy(const Weights &to) : GenericTrieUnigramProxy<Weights>(to) {}
- TrieUnigramProxy() {}
- float Rest() const { return to_->rest; }
- };
-
-// gcc 4.1 doesn't properly back dependent types :-(.
-#pragma pack(push)
-#pragma pack(4)
- struct ProbingEntry {
- typedef uint64_t Key;
- typedef Weights Value;
- Key key;
- Value value;
- Key GetKey() const { return key; }
- };
-
- struct TrieUnigramValue {
- Weights weights;
- uint64_t next;
- uint64_t Next() const { return next; }
- };
-#pragma pack(pop)
-
- const static bool kDifferentRest = true;
-
- template <class Model, class C> void Callback(const Config &config, unsigned int order, typename Model::Vocabulary &vocab, C &callback) {
- switch (config.rest_function) {
- case Config::REST_MAX:
- {
- MaxRestBuild build;
- callback(build);
- }
- break;
- case Config::REST_LOWER:
- {
- LowerRestBuild<Model> build(config, order, vocab);
- callback(build);
- }
- break;
- }
- }
-};
-
-} // namespace ngram
-} // namespace lm
-
-#endif // LM_VALUE_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/value_build.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/value_build.cc b/ext/kenlm/lm/value_build.cc
deleted file mode 100644
index ac623a6..0000000
--- a/ext/kenlm/lm/value_build.cc
+++ /dev/null
@@ -1,59 +0,0 @@
-#include "lm/value_build.hh"
-
-#include "lm/model.hh"
-#include "lm/read_arpa.hh"
-
-namespace lm {
-namespace ngram {
-
-template <class Model> LowerRestBuild<Model>::LowerRestBuild(const Config &config, unsigned int order, const typename Model::Vocabulary &vocab) {
- UTIL_THROW_IF(config.rest_lower_files.size() != order - 1, ConfigException, "This model has order " << order << " so there should be " << (order - 1) << " lower-order models for rest cost purposes.");
- Config for_lower = config;
- for_lower.write_mmap = NULL;
- for_lower.rest_lower_files.clear();
-
- // Unigram models aren't supported, so this is a custom loader.
- // TODO: optimize the unigram loading?
- {
- util::FilePiece uni(config.rest_lower_files[0].c_str());
- std::vector<uint64_t> number;
- ReadARPACounts(uni, number);
- UTIL_THROW_IF(number.size() != 1, FormatLoadException, "Expected the unigram model to have order 1, not " << number.size());
- ReadNGramHeader(uni, 1);
- unigrams_.resize(number[0]);
- unigrams_[0] = config.unknown_missing_logprob;
- PositiveProbWarn warn;
- for (uint64_t i = 0; i < number[0]; ++i) {
- WordIndex w;
- Prob entry;
- ReadNGram(uni, 1, vocab, &w, entry, warn);
- unigrams_[w] = entry.prob;
- }
- }
-
- try {
- for (unsigned int i = 2; i < order; ++i) {
- models_.push_back(new Model(config.rest_lower_files[i - 1].c_str(), for_lower));
- UTIL_THROW_IF(models_.back()->Order() != i, FormatLoadException, "Lower order file " << config.rest_lower_files[i-1] << " should have order " << i);
- }
- } catch (...) {
- for (typename std::vector<const Model*>::const_iterator i = models_.begin(); i != models_.end(); ++i) {
- delete *i;
- }
- models_.clear();
- throw;
- }
-
- // TODO: force/check same vocab.
-}
-
-template <class Model> LowerRestBuild<Model>::~LowerRestBuild() {
- for (typename std::vector<const Model*>::const_iterator i = models_.begin(); i != models_.end(); ++i) {
- delete *i;
- }
-}
-
-template class LowerRestBuild<ProbingModel>;
-
-} // namespace ngram
-} // namespace lm
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/value_build.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/value_build.hh b/ext/kenlm/lm/value_build.hh
deleted file mode 100644
index 49989ab..0000000
--- a/ext/kenlm/lm/value_build.hh
+++ /dev/null
@@ -1,97 +0,0 @@
-#ifndef LM_VALUE_BUILD_H
-#define LM_VALUE_BUILD_H
-
-#include "lm/weights.hh"
-#include "lm/word_index.hh"
-#include "util/bit_packing.hh"
-
-#include <vector>
-
-namespace lm {
-namespace ngram {
-
-struct Config;
-struct BackoffValue;
-struct RestValue;
-
-class NoRestBuild {
- public:
- typedef BackoffValue Value;
-
- NoRestBuild() {}
-
- void SetRest(const WordIndex *, unsigned int, const Prob &/*prob*/) const {}
- void SetRest(const WordIndex *, unsigned int, const ProbBackoff &) const {}
-
- template <class Second> bool MarkExtends(ProbBackoff &weights, const Second &) const {
- util::UnsetSign(weights.prob);
- return false;
- }
-
- // Probing doesn't need to go back to unigram.
- const static bool kMarkEvenLower = false;
-};
-
-class MaxRestBuild {
- public:
- typedef RestValue Value;
-
- MaxRestBuild() {}
-
- void SetRest(const WordIndex *, unsigned int, const Prob &/*prob*/) const {}
- void SetRest(const WordIndex *, unsigned int, RestWeights &weights) const {
- weights.rest = weights.prob;
- util::SetSign(weights.rest);
- }
-
- bool MarkExtends(RestWeights &weights, const RestWeights &to) const {
- util::UnsetSign(weights.prob);
- if (weights.rest >= to.rest) return false;
- weights.rest = to.rest;
- return true;
- }
- bool MarkExtends(RestWeights &weights, const Prob &to) const {
- util::UnsetSign(weights.prob);
- if (weights.rest >= to.prob) return false;
- weights.rest = to.prob;
- return true;
- }
-
- // Probing does need to go back to unigram.
- const static bool kMarkEvenLower = true;
-};
-
-template <class Model> class LowerRestBuild {
- public:
- typedef RestValue Value;
-
- LowerRestBuild(const Config &config, unsigned int order, const typename Model::Vocabulary &vocab);
-
- ~LowerRestBuild();
-
- void SetRest(const WordIndex *, unsigned int, const Prob &/*prob*/) const {}
- void SetRest(const WordIndex *vocab_ids, unsigned int n, RestWeights &weights) const {
- typename Model::State ignored;
- if (n == 1) {
- weights.rest = unigrams_[*vocab_ids];
- } else {
- weights.rest = models_[n-2]->FullScoreForgotState(vocab_ids + 1, vocab_ids + n, *vocab_ids, ignored).prob;
- }
- }
-
- template <class Second> bool MarkExtends(RestWeights &weights, const Second &) const {
- util::UnsetSign(weights.prob);
- return false;
- }
-
- const static bool kMarkEvenLower = false;
-
- std::vector<float> unigrams_;
-
- std::vector<const Model*> models_;
-};
-
-} // namespace ngram
-} // namespace lm
-
-#endif // LM_VALUE_BUILD_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/virtual_interface.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/virtual_interface.cc b/ext/kenlm/lm/virtual_interface.cc
deleted file mode 100644
index 17a74c3..0000000
--- a/ext/kenlm/lm/virtual_interface.cc
+++ /dev/null
@@ -1,19 +0,0 @@
-#include "lm/virtual_interface.hh"
-
-#include "lm/lm_exception.hh"
-
-namespace lm {
-namespace base {
-
-Vocabulary::~Vocabulary() {}
-
-void Vocabulary::SetSpecial(WordIndex begin_sentence, WordIndex end_sentence, WordIndex not_found) {
- begin_sentence_ = begin_sentence;
- end_sentence_ = end_sentence;
- not_found_ = not_found;
-}
-
-Model::~Model() {}
-
-} // namespace base
-} // namespace lm
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/lm/virtual_interface.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/lm/virtual_interface.hh b/ext/kenlm/lm/virtual_interface.hh
deleted file mode 100644
index ea491fb..0000000
--- a/ext/kenlm/lm/virtual_interface.hh
+++ /dev/null
@@ -1,160 +0,0 @@
-#ifndef LM_VIRTUAL_INTERFACE_H
-#define LM_VIRTUAL_INTERFACE_H
-
-#include "lm/return.hh"
-#include "lm/word_index.hh"
-#include "util/string_piece.hh"
-
-#include <string>
-#include <cstring>
-
-namespace lm {
-namespace base {
-
-template <class T, class U, class V> class ModelFacade;
-
-/* Vocabulary interface. Call Index(string) and get a word index for use in
- * calling Model. It provides faster convenience functions for <s>, </s>, and
- * <unk> although you can also find these using Index.
- *
- * Some models do not load the mapping from index to string. If you need this,
- * check if the model Vocabulary class implements such a function and access it
- * directly.
- *
- * The Vocabulary object is always owned by the Model and can be retrieved from
- * the Model using BaseVocabulary() for this abstract interface or
- * GetVocabulary() for the actual implementation (in which case you'll need the
- * actual implementation of the Model too).
- */
-class Vocabulary {
- public:
- virtual ~Vocabulary();
-
- WordIndex BeginSentence() const { return begin_sentence_; }
- WordIndex EndSentence() const { return end_sentence_; }
- WordIndex NotFound() const { return not_found_; }
-
- /* Most implementations allow StringPiece lookups and need only override
- * Index(StringPiece). SRI requires null termination and overrides all
- * three methods.
- */
- virtual WordIndex Index(const StringPiece &str) const = 0;
- virtual WordIndex Index(const std::string &str) const {
- return Index(StringPiece(str));
- }
- virtual WordIndex Index(const char *str) const {
- return Index(StringPiece(str));
- }
-
- protected:
- // Call SetSpecial afterward.
- Vocabulary() {}
-
- Vocabulary(WordIndex begin_sentence, WordIndex end_sentence, WordIndex not_found) {
- SetSpecial(begin_sentence, end_sentence, not_found);
- }
-
- void SetSpecial(WordIndex begin_sentence, WordIndex end_sentence, WordIndex not_found);
-
- WordIndex begin_sentence_, end_sentence_, not_found_;
-
- private:
- // Disable copy constructors. They're private and undefined.
- // Ersatz boost::noncopyable.
- Vocabulary(const Vocabulary &);
- Vocabulary &operator=(const Vocabulary &);
-};
-
-/* There are two ways to access a Model.
- *
- *
- * OPTION 1: Access the Model directly (e.g. lm::ngram::Model in model.hh).
- *
- * Every Model implements the scoring function:
- * float Score(
- * const Model::State &in_state,
- * const WordIndex new_word,
- * Model::State &out_state) const;
- *
- * It can also return the length of n-gram matched by the model:
- * FullScoreReturn FullScore(
- * const Model::State &in_state,
- * const WordIndex new_word,
- * Model::State &out_state) const;
- *
- *
- * There are also accessor functions:
- * const State &BeginSentenceState() const;
- * const State &NullContextState() const;
- * const Vocabulary &GetVocabulary() const;
- * unsigned int Order() const;
- *
- * NB: In case you're wondering why the model implementation looks like it's
- * missing these methods, see facade.hh.
- *
- * This is the fastest way to use a model and presents a normal State class to
- * be included in a hypothesis state structure.
- *
- *
- * OPTION 2: Use the virtual interface below.
- *
- * The virtual interface allow you to decide which Model to use at runtime
- * without templatizing everything on the Model type. However, each Model has
- * its own State class, so a single State cannot be efficiently provided (it
- * would require using the maximum memory of any Model's State or memory
- * allocation with each lookup). This means you become responsible for
- * allocating memory with size StateSize() and passing it to the Score or
- * FullScore functions provided here.
- *
- * For example, cdec has a std::string containing the entire state of a
- * hypothesis. It can reserve StateSize bytes in this string for the model
- * state.
- *
- * All the State objects are POD, so it's ok to use raw memory for storing
- * State.
- * in_state and out_state must not have the same address.
- */
-class Model {
- public:
- virtual ~Model();
-
- size_t StateSize() const { return state_size_; }
- const void *BeginSentenceMemory() const { return begin_sentence_memory_; }
- void BeginSentenceWrite(void *to) const { memcpy(to, begin_sentence_memory_, StateSize()); }
- const void *NullContextMemory() const { return null_context_memory_; }
- void NullContextWrite(void *to) const { memcpy(to, null_context_memory_, StateSize()); }
-
- // Requires in_state != out_state
- virtual float BaseScore(const void *in_state, const WordIndex new_word, void *out_state) const = 0;
-
- // Requires in_state != out_state
- virtual FullScoreReturn BaseFullScore(const void *in_state, const WordIndex new_word, void *out_state) const = 0;
-
- // Prefer to use FullScore. The context words should be provided in reverse order.
- virtual FullScoreReturn BaseFullScoreForgotState(const WordIndex *context_rbegin, const WordIndex *context_rend, const WordIndex new_word, void *out_state) const = 0;
-
- unsigned char Order() const { return order_; }
-
- const Vocabulary &BaseVocabulary() const { return *base_vocab_; }
-
- private:
- template <class T, class U, class V> friend class ModelFacade;
- explicit Model(size_t state_size) : state_size_(state_size) {}
-
- const size_t state_size_;
- const void *begin_sentence_memory_, *null_context_memory_;
-
- const Vocabulary *base_vocab_;
-
- unsigned char order_;
-
- // Disable copy constructors. They're private and undefined.
- // Ersatz boost::noncopyable.
- Model(const Model &);
- Model &operator=(const Model &);
-};
-
-} // mamespace base
-} // namespace lm
-
-#endif // LM_VIRTUAL_INTERFACE_H
[18/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/modules/order.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/modules/order.c b/ext/kenlm/jam-files/engine/modules/order.c
deleted file mode 100644
index 3a83d38..0000000
--- a/ext/kenlm/jam-files/engine/modules/order.c
+++ /dev/null
@@ -1,160 +0,0 @@
-/* Copyright 2004. Vladimir Prus
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#include "../lists.h"
-#include "../mem.h"
-#include "../native.h"
-#include "../object.h"
-#include "../strings.h"
-#include "../variable.h"
-
-
-/* Use quite klugy approach: when we add order dependency from 'a' to 'b', just
- * append 'b' to of value of variable 'a'.
- */
-LIST * add_pair( FRAME * frame, int flags )
-{
- LIST * arg = lol_get( frame->args, 0 );
- LISTITER iter = list_begin( arg );
- LISTITER const end = list_end( arg );
- var_set( frame->module, list_item( iter ), list_copy_range( arg, list_next(
- iter ), end ), VAR_APPEND );
- return L0;
-}
-
-
-/* Given a list and a value, returns position of that value in the list, or -1
- * if not found.
- */
-int list_index( LIST * list, OBJECT * value )
-{
- int result = 0;
- LISTITER iter = list_begin( list );
- LISTITER const end = list_end( list );
- for ( ; iter != end; iter = list_next( iter ), ++result )
- if ( object_equal( list_item( iter ), value ) )
- return result;
- return -1;
-}
-
-enum colors { white, gray, black };
-
-
-/* Main routine for topological sort. Calls itself recursively on all adjacent
- * vertices which were not yet visited. After that, 'current_vertex' is added to
- * '*result_ptr'.
- */
-void do_ts( int * * graph, int current_vertex, int * colors, int * * result_ptr
- )
-{
- int i;
-
- colors[ current_vertex ] = gray;
- for ( i = 0; graph[ current_vertex ][ i ] != -1; ++i )
- {
- int adjacent_vertex = graph[ current_vertex ][ i ];
- if ( colors[ adjacent_vertex ] == white )
- do_ts( graph, adjacent_vertex, colors, result_ptr );
- /* The vertex is either black, in which case we do not have to do
- * anything, or gray, in which case we have a loop. If we have a loop,
- * it is not clear what useful diagnostic we can emit, so we emit
- * nothing.
- */
- }
- colors[ current_vertex ] = black;
- **result_ptr = current_vertex;
- ( *result_ptr )++;
-}
-
-
-void topological_sort( int * * graph, int num_vertices, int * result )
-{
- int i;
- int * colors = ( int * )BJAM_CALLOC( num_vertices, sizeof( int ) );
- for ( i = 0; i < num_vertices; ++i )
- colors[ i ] = white;
-
- for ( i = 0; i < num_vertices; ++i )
- if ( colors[ i ] == white )
- do_ts( graph, i, colors, &result );
-
- BJAM_FREE( colors );
-}
-
-
-LIST * order( FRAME * frame, int flags )
-{
- LIST * arg = lol_get( frame->args, 0 );
- LIST * result = L0;
- int src;
- LISTITER iter = list_begin( arg );
- LISTITER const end = list_end( arg );
-
- /* We need to create a graph of order dependencies between the passed
- * objects. We assume there are no duplicates passed to 'add_pair'.
- */
- int length = list_length( arg );
- int * * graph = ( int * * )BJAM_CALLOC( length, sizeof( int * ) );
- int * order = ( int * )BJAM_MALLOC( ( length + 1 ) * sizeof( int ) );
-
- for ( src = 0; iter != end; iter = list_next( iter ), ++src )
- {
- /* For all objects this one depends upon, add elements to 'graph'. */
- LIST * dependencies = var_get( frame->module, list_item( iter ) );
- int index = 0;
- LISTITER dep_iter = list_begin( dependencies );
- LISTITER const dep_end = list_end( dependencies );
-
- graph[ src ] = ( int * )BJAM_CALLOC( list_length( dependencies ) + 1,
- sizeof( int ) );
- for ( ; dep_iter != dep_end; dep_iter = list_next( dep_iter ) )
- {
- int const dst = list_index( arg, list_item( dep_iter ) );
- if ( dst != -1 )
- graph[ src ][ index++ ] = dst;
- }
- graph[ src ][ index ] = -1;
- }
-
- topological_sort( graph, length, order );
-
- {
- int index = length - 1;
- for ( ; index >= 0; --index )
- {
- int i;
- LISTITER iter = list_begin( arg );
- LISTITER const end = list_end( arg );
- for ( i = 0; i < order[ index ]; ++i, iter = list_next( iter ) );
- result = list_push_back( result, object_copy( list_item( iter ) ) );
- }
- }
-
- /* Clean up */
- {
- int i;
- for ( i = 0; i < length; ++i )
- BJAM_FREE( graph[ i ] );
- BJAM_FREE( graph );
- BJAM_FREE( order );
- }
-
- return result;
-}
-
-
-void init_order()
-{
- {
- char const * args[] = { "first", "second", 0 };
- declare_native_rule( "class@order", "add-pair", args, add_pair, 1 );
- }
-
- {
- char const * args[] = { "objects", "*", 0 };
- declare_native_rule( "class@order", "order", args, order, 1 );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/modules/path.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/modules/path.c b/ext/kenlm/jam-files/engine/modules/path.c
deleted file mode 100644
index f8dedac..0000000
--- a/ext/kenlm/jam-files/engine/modules/path.c
+++ /dev/null
@@ -1,25 +0,0 @@
-/* Copyright Vladimir Prus 2003.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#include "../constants.h"
-#include "../frames.h"
-#include "../lists.h"
-#include "../native.h"
-#include "../timestamp.h"
-
-
-LIST * path_exists( FRAME * frame, int flags )
-{
- return file_query( list_front( lol_get( frame->args, 0 ) ) ) ?
- list_new( object_copy( constant_true ) ) : L0;
-}
-
-
-void init_path()
-{
- char const * args[] = { "location", 0 };
- declare_native_rule( "path", "exists", args, path_exists, 1 );
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/modules/property-set.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/modules/property-set.c b/ext/kenlm/jam-files/engine/modules/property-set.c
deleted file mode 100644
index 21e35d5..0000000
--- a/ext/kenlm/jam-files/engine/modules/property-set.c
+++ /dev/null
@@ -1,330 +0,0 @@
-/*
- * Copyright 2013 Steven Watanabe
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#include "../object.h"
-#include "../lists.h"
-#include "../modules.h"
-#include "../rules.h"
-#include "../variable.h"
-#include "../native.h"
-#include "../compile.h"
-#include "../mem.h"
-#include "../constants.h"
-#include "string.h"
-
-struct ps_map_entry
-{
- struct ps_map_entry * next;
- LIST * key;
- OBJECT * value;
-};
-
-struct ps_map
-{
- struct ps_map_entry * * table;
- size_t table_size;
- size_t num_elems;
-};
-
-static unsigned list_hash(LIST * key)
-{
- unsigned int hash = 0;
- LISTITER iter = list_begin( key ), end = list_end( key );
- for ( ; iter != end; ++iter )
- {
- hash = hash * 2147059363 + object_hash( list_item( iter ) );
- }
- return hash;
-}
-
-static int list_equal( LIST * lhs, LIST * rhs )
-{
- LISTITER lhs_iter, lhs_end, rhs_iter;
- if ( list_length( lhs ) != list_length( rhs ) )
- {
- return 0;
- }
- lhs_iter = list_begin( lhs );
- lhs_end = list_end( lhs );
- rhs_iter = list_begin( rhs );
- for ( ; lhs_iter != lhs_end; ++lhs_iter, ++rhs_iter )
- {
- if ( ! object_equal( list_item( lhs_iter ), list_item( rhs_iter ) ) )
- {
- return 0;
- }
- }
- return 1;
-}
-
-static void ps_map_init( struct ps_map * map )
-{
- size_t i;
- map->table_size = 2;
- map->num_elems = 0;
- map->table = BJAM_MALLOC( map->table_size * sizeof( struct ps_map_entry * ) );
- for ( i = 0; i < map->table_size; ++i )
- {
- map->table[ i ] = NULL;
- }
-}
-
-static void ps_map_destroy( struct ps_map * map )
-{
- size_t i;
- for ( i = 0; i < map->table_size; ++i )
- {
- struct ps_map_entry * pos;
- for ( pos = map->table[ i ]; pos; )
- {
- struct ps_map_entry * tmp = pos->next;
- BJAM_FREE( pos );
- pos = tmp;
- }
- }
- BJAM_FREE( map->table );
-}
-
-static void ps_map_rehash( struct ps_map * map )
-{
- struct ps_map old = *map;
- size_t i;
- map->table = BJAM_MALLOC( map->table_size * 2 * sizeof( struct ps_map_entry * ) );
- map->table_size *= 2;
- for ( i = 0; i < map->table_size; ++i )
- {
- map->table[ i ] = NULL;
- }
- for ( i = 0; i < old.table_size; ++i )
- {
- struct ps_map_entry * pos;
- for ( pos = old.table[ i ]; pos; )
- {
- struct ps_map_entry * tmp = pos->next;
-
- unsigned hash_val = list_hash( pos->key );
- unsigned bucket = hash_val % map->table_size;
- pos->next = map->table[ bucket ];
- map->table[ bucket ] = pos;
-
- pos = tmp;
- }
- }
- BJAM_FREE( old.table );
-}
-
-static struct ps_map_entry * ps_map_insert(struct ps_map * map, LIST * key)
-{
- unsigned hash_val = list_hash( key );
- unsigned bucket = hash_val % map->table_size;
- struct ps_map_entry * pos;
- for ( pos = map->table[bucket]; pos ; pos = pos->next )
- {
- if ( list_equal( pos->key, key ) )
- return pos;
- }
-
- if ( map->num_elems >= map->table_size )
- {
- ps_map_rehash( map );
- bucket = hash_val % map->table_size;
- }
- pos = BJAM_MALLOC( sizeof( struct ps_map_entry ) );
- pos->next = map->table[bucket];
- pos->key = key;
- pos->value = 0;
- map->table[bucket] = pos;
- ++map->num_elems;
- return pos;
-}
-
-static struct ps_map all_property_sets;
-
-LIST * property_set_create( FRAME * frame, int flags )
-{
- LIST * properties = lol_get( frame->args, 0 );
- LIST * sorted = list_sort( properties );
- LIST * unique = list_unique( sorted );
- struct ps_map_entry * pos = ps_map_insert( &all_property_sets, unique );
- list_free( sorted );
- if ( pos->value )
- {
- list_free( unique );
- return list_new( object_copy( pos->value ) );
- }
- else
- {
- OBJECT * rulename = object_new( "new" );
- OBJECT * varname = object_new( "self.raw" );
- LIST * val = call_rule( rulename, frame,
- list_new( object_new( "property-set" ) ), 0 );
- LISTITER iter, end;
- object_free( rulename );
- pos->value = list_front( val );
- var_set( bindmodule( pos->value ), varname, unique, VAR_SET );
- object_free( varname );
-
- for ( iter = list_begin( unique ), end = list_end( unique ); iter != end; ++iter )
- {
- const char * str = object_str( list_item( iter ) );
- if ( str[ 0 ] != '<' || ! strchr( str, '>' ) )
- {
- string message[ 1 ];
- string_new( message );
- string_append( message, "Invalid property: '" );
- string_append( message, str );
- string_append( message, "'" );
- rulename = object_new( "errors.error" );
- call_rule( rulename, frame,
- list_new( object_new( message->value ) ), 0 );
- /* unreachable */
- string_free( message );
- object_free( rulename );
- }
- }
-
- return val;
- }
-}
-
-/* binary search for the property value */
-LIST * property_set_get( FRAME * frame, int flags )
-{
- OBJECT * varname = object_new( "self.raw" );
- LIST * props = var_get( frame->module, varname );
- const char * name = object_str( list_front( lol_get( frame->args, 0 ) ) );
- size_t name_len = strlen( name );
- LISTITER begin, end;
- LIST * result = L0;
- object_free( varname );
-
- /* Assumes random access */
- begin = list_begin( props ), end = list_end( props );
-
- while ( 1 )
- {
- ptrdiff_t diff = (end - begin);
- LISTITER mid = begin + diff / 2;
- int res;
- if ( diff == 0 )
- {
- return L0;
- }
- res = strncmp( object_str( list_item( mid ) ), name, name_len );
- if ( res < 0 )
- {
- begin = mid + 1;
- }
- else if ( res > 0 )
- {
- end = mid;
- }
- else /* We've found the property */
- {
- /* Find the beginning of the group */
- LISTITER tmp = mid;
- while ( tmp > begin )
- {
- --tmp;
- res = strncmp( object_str( list_item( tmp ) ), name, name_len );
- if ( res != 0 )
- {
- ++tmp;
- break;
- }
- }
- begin = tmp;
- /* Find the end of the group */
- tmp = mid + 1;
- while ( tmp < end )
- {
- res = strncmp( object_str( list_item( tmp ) ), name, name_len );
- if ( res != 0 ) break;
- ++tmp;
- }
- end = tmp;
- break;
- }
- }
-
- for ( ; begin != end; ++begin )
- {
- result = list_push_back( result,
- object_new( object_str( list_item( begin ) ) + name_len ) );
- }
-
- return result;
-}
-
-/* binary search for the property value */
-LIST * property_set_contains_features( FRAME * frame, int flags )
-{
- OBJECT * varname = object_new( "self.raw" );
- LIST * props = var_get( frame->module, varname );
- LIST * features = lol_get( frame->args, 0 );
- LIST * result = L0;
- LISTITER features_iter = list_begin( features );
- LISTITER features_end = list_end( features ) ;
- object_free( varname );
-
- for ( ; features_iter != features_end; ++features_iter )
- {
- const char * name = object_str( list_item( features_iter ) );
- size_t name_len = strlen( name );
- LISTITER begin, end;
- /* Assumes random access */
- begin = list_begin( props ), end = list_end( props );
-
- while ( 1 )
- {
- ptrdiff_t diff = (end - begin);
- LISTITER mid = begin + diff / 2;
- int res;
- if ( diff == 0 )
- {
- /* The feature is missing */
- return L0;
- }
- res = strncmp( object_str( list_item( mid ) ), name, name_len );
- if ( res < 0 )
- {
- begin = mid + 1;
- }
- else if ( res > 0 )
- {
- end = mid;
- }
- else /* We've found the property */
- {
- break;
- }
- }
- }
- return list_new( object_copy( constant_true ) );
-}
-
-void init_property_set()
-{
- {
- char const * args[] = { "raw-properties", "*", 0 };
- declare_native_rule( "property-set", "create", args, property_set_create, 1 );
- }
- {
- char const * args[] = { "feature", 0 };
- declare_native_rule( "class@property-set", "get", args, property_set_get, 1 );
- }
- {
- char const * args[] = { "features", "*", 0 };
- declare_native_rule( "class@property-set", "contains-features", args, property_set_contains_features, 1 );
- }
- ps_map_init( &all_property_sets );
-}
-
-void property_set_done()
-{
- ps_map_destroy( &all_property_sets );
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/modules/readme.txt
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/modules/readme.txt b/ext/kenlm/jam-files/engine/modules/readme.txt
deleted file mode 100644
index 2edf6e1..0000000
--- a/ext/kenlm/jam-files/engine/modules/readme.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-
-This directory constains sources which declare native
-rules for Boost.Build modules.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/modules/regex.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/modules/regex.c b/ext/kenlm/jam-files/engine/modules/regex.c
deleted file mode 100644
index d9f8177..0000000
--- a/ext/kenlm/jam-files/engine/modules/regex.c
+++ /dev/null
@@ -1,220 +0,0 @@
-/*
- * Copyright 2003. Vladimir Prus
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#include "../mem.h"
-#include "../native.h"
-#include "../strings.h"
-#include "../subst.h"
-
-/*
-rule split ( string separator )
-{
- local result ;
- local s = $(string) ;
-
- local match = 1 ;
- while $(match)
- {
- match = [ MATCH ^(.*)($(separator))(.*) : $(s) ] ;
- if $(match)
- {
- match += "" ; # in case 3rd item was empty - works around MATCH bug
- result = $(match[3]) $(result) ;
- s = $(match[1]) ;
- }
- }
- return $(s) $(result) ;
-}
-*/
-
-LIST * regex_split( FRAME * frame, int flags )
-{
- LIST * args = lol_get( frame->args, 0 );
- OBJECT * s;
- OBJECT * separator;
- regexp * re;
- const char * pos;
- LIST * result = L0;
- LISTITER iter = list_begin( args );
- s = list_item( iter );
- separator = list_item( list_next( iter ) );
-
- re = regex_compile( separator );
-
- pos = object_str( s );
- while ( regexec( re, pos ) )
- {
- result = list_push_back( result, object_new_range( pos, re->startp[ 0 ] - pos ) );
- pos = re->endp[ 0 ];
- }
-
- result = list_push_back( result, object_new( pos ) );
-
- return result;
-}
-
-/*
-rule replace (
- string # The string to modify.
- match # The characters to replace.
- replacement # The string to replace with.
- )
-{
- local result = "" ;
- local parts = 1 ;
- while $(parts)
- {
- parts = [ MATCH ^(.*)($(match))(.*) : $(string) ] ;
- if $(parts)
- {
- parts += "" ;
- result = "$(replacement)$(parts[3])$(result)" ;
- string = $(parts[1]) ;
- }
- }
- string ?= "" ;
- result = "$(string)$(result)" ;
- return $(result) ;
-}
-*/
-
-LIST * regex_replace( FRAME * frame, int flags )
-{
- LIST * args = lol_get( frame->args, 0 );
- OBJECT * s;
- OBJECT * match;
- OBJECT * replacement;
- regexp * re;
- const char * pos;
- string buf[ 1 ];
- LIST * result;
- LISTITER iter = list_begin( args );
- s = list_item( iter );
- iter = list_next( iter );
- match = list_item( iter );
- iter = list_next( iter );
- replacement = list_item(iter );
-
- re = regex_compile( match );
-
- string_new( buf );
-
- pos = object_str( s );
- while ( regexec( re, pos ) )
- {
- string_append_range( buf, pos, re->startp[ 0 ] );
- string_append( buf, object_str( replacement ) );
- pos = re->endp[ 0 ];
- }
- string_append( buf, pos );
-
- result = list_new( object_new( buf->value ) );
-
- string_free( buf );
-
- return result;
-}
-
-/*
-rule transform ( list * : pattern : indices * )
-{
- indices ?= 1 ;
- local result ;
- for local e in $(list)
- {
- local m = [ MATCH $(pattern) : $(e) ] ;
- if $(m)
- {
- result += $(m[$(indices)]) ;
- }
- }
- return $(result) ;
-}
-*/
-
-LIST * regex_transform( FRAME * frame, int flags )
-{
- LIST * const l = lol_get( frame->args, 0 );
- LIST * const pattern = lol_get( frame->args, 1 );
- LIST * const indices_list = lol_get( frame->args, 2 );
- int * indices = 0;
- int size;
- LIST * result = L0;
-
- if ( !list_empty( indices_list ) )
- {
- int * p;
- LISTITER iter = list_begin( indices_list );
- LISTITER const end = list_end( indices_list );
- size = list_length( indices_list );
- indices = (int *)BJAM_MALLOC( size * sizeof( int ) );
- for ( p = indices; iter != end; iter = list_next( iter ) )
- *p++ = atoi( object_str( list_item( iter ) ) );
- }
- else
- {
- size = 1;
- indices = (int *)BJAM_MALLOC( sizeof( int ) );
- *indices = 1;
- }
-
- {
- /* Result is cached and intentionally never freed */
- regexp * const re = regex_compile( list_front( pattern ) );
-
- LISTITER iter = list_begin( l );
- LISTITER const end = list_end( l );
-
- string buf[ 1 ];
- string_new( buf );
-
- for ( ; iter != end; iter = list_next( iter ) )
- {
- if ( regexec( re, object_str( list_item( iter ) ) ) )
- {
- int i = 0;
- for ( ; i < size; ++i )
- {
- int const index = indices[ i ];
- /* Skip empty submatches. Not sure it is right in all cases,
- * but surely is right for the case for which this routine
- * is optimized -- header scanning.
- */
- if ( re->startp[ index ] != re->endp[ index ] )
- {
- string_append_range( buf, re->startp[ index ],
- re->endp[ index ] );
- result = list_push_back( result, object_new( buf->value
- ) );
- string_truncate( buf, 0 );
- }
- }
- }
- }
- string_free( buf );
- }
-
- BJAM_FREE( indices );
- return result;
-}
-
-
-void init_regex()
-{
- {
- char const * args[] = { "string", "separator", 0 };
- declare_native_rule( "regex", "split", args, regex_split, 1 );
- }
- {
- char const * args[] = { "string", "match", "replacement", 0 };
- declare_native_rule( "regex", "replace", args, regex_replace, 1 );
- }
- {
- char const * args[] = { "list", "*", ":", "pattern", ":", "indices", "*", 0 };
- declare_native_rule( "regex", "transform", args, regex_transform, 2 );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/modules/sequence.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/modules/sequence.c b/ext/kenlm/jam-files/engine/modules/sequence.c
deleted file mode 100644
index 08ed305..0000000
--- a/ext/kenlm/jam-files/engine/modules/sequence.c
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Copyright Vladimir Prus 2003.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#include "../native.h"
-#include "../object.h"
-#include "../lists.h"
-#include "../compile.h"
-
-#include <stdlib.h>
-
-
-#ifndef max
-# define max(a,b) ((a)>(b)?(a):(b))
-#endif
-
-
-LIST * sequence_select_highest_ranked( FRAME * frame, int flags )
-{
- /* Returns all of 'elements' for which corresponding element in parallel */
- /* list 'rank' is equal to the maximum value in 'rank'. */
-
- LIST * const elements = lol_get( frame->args, 0 );
- LIST * const rank = lol_get( frame->args, 1 );
-
- LIST * result = L0;
- int highest_rank = -1;
-
- {
- LISTITER iter = list_begin( rank );
- LISTITER const end = list_end( rank );
- for ( ; iter != end; iter = list_next( iter ) )
- {
- int const current = atoi( object_str( list_item( iter ) ) );
- highest_rank = max( highest_rank, current );
- }
- }
-
- {
- LISTITER iter = list_begin( rank );
- LISTITER const end = list_end( rank );
- LISTITER elements_iter = list_begin( elements );
- LISTITER const elements_end = list_end( elements );
- for ( ; iter != end; iter = list_next( iter ), elements_iter =
- list_next( elements_iter ) )
- if ( atoi( object_str( list_item( iter ) ) ) == highest_rank )
- result = list_push_back( result, object_copy( list_item(
- elements_iter ) ) );
- }
-
- return result;
-}
-
-LIST * sequence_transform( FRAME * frame, int flags )
-{
- LIST * function = lol_get( frame->args, 0 );
- LIST * sequence = lol_get( frame->args, 1 );
- LIST * result = L0;
- OBJECT * function_name = list_front( function );
- LISTITER args_begin = list_next( list_begin( function ) ), args_end = list_end( function );
- LISTITER iter = list_begin( sequence ), end = list_end( sequence );
- RULE * rule = bindrule( function_name, frame->prev->module );
-
- for ( ; iter != end; iter = list_next( iter ) )
- {
- FRAME inner[ 1 ];
-
- frame_init( inner );
- inner->prev = frame;
- inner->prev_user = frame->prev_user;
- inner->module = frame->prev->module;
-
- lol_add( inner->args, list_push_back( list_copy_range( function, args_begin, args_end ), object_copy( list_item( iter ) ) ) );
- result = list_append( result, evaluate_rule( rule, function_name, inner ) );
-
- frame_free( inner );
- }
-
- return result;
-}
-
-void init_sequence()
-{
- {
- char const * args[] = { "elements", "*", ":", "rank", "*", 0 };
- declare_native_rule( "sequence", "select-highest-ranked", args,
- sequence_select_highest_ranked, 1 );
- }
- {
- char const * args[] = { "function", "+", ":", "sequence", "*", 0 };
- declare_native_rule( "sequence", "transform", args,
- sequence_transform, 1 );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/modules/set.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/modules/set.c b/ext/kenlm/jam-files/engine/modules/set.c
deleted file mode 100644
index 77a314d..0000000
--- a/ext/kenlm/jam-files/engine/modules/set.c
+++ /dev/null
@@ -1,43 +0,0 @@
-/* Copyright Vladimir Prus 2003. Distributed under the Boost */
-/* Software License, Version 1.0. (See accompanying */
-/* file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) */
-
-#include "../native.h"
-#include "../object.h"
-
-/*
- local result = ;
- local element ;
- for element in $(B)
- {
- if ! ( $(element) in $(A) )
- {
- result += $(element) ;
- }
- }
- return $(result) ;
-*/
-LIST *set_difference( FRAME *frame, int flags )
-{
-
- LIST* b = lol_get( frame->args, 0 );
- LIST* a = lol_get( frame->args, 1 );
-
- LIST* result = L0;
- LISTITER iter = list_begin( b ), end = list_end( b );
- for( ; iter != end; iter = list_next( iter ) )
- {
- if (!list_in(a, list_item(iter)))
- result = list_push_back(result, object_copy(list_item(iter)));
- }
- return result;
-}
-
-void init_set()
-{
- {
- const char* args[] = { "B", "*", ":", "A", "*", 0 };
- declare_native_rule("set", "difference", args, set_difference, 1);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/native.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/native.c b/ext/kenlm/jam-files/engine/native.c
deleted file mode 100644
index 68828aa..0000000
--- a/ext/kenlm/jam-files/engine/native.c
+++ /dev/null
@@ -1,34 +0,0 @@
-/* Copyright 2003. Vladimir Prus
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#include "native.h"
-
-#include "hash.h"
-
-#include <assert.h>
-
-
-void declare_native_rule( char const * module, char const * rule,
- char const * * args, LIST * (*f)( FRAME *, int ), int version )
-{
- OBJECT * const module_obj = module ? object_new( module ) : 0 ;
- module_t * m = bindmodule( module_obj );
- if ( module_obj )
- object_free( module_obj );
- if ( !m->native_rules )
- m->native_rules = hashinit( sizeof( native_rule_t ), "native rules" );
-
- {
- OBJECT * const name = object_new( rule );
- int found;
- native_rule_t * const np = (native_rule_t *)hash_insert(
- m->native_rules, name, &found );
- np->name = name;
- assert( !found );
- np->procedure = function_builtin( f, 0, args );
- np->version = version;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/native.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/native.h b/ext/kenlm/jam-files/engine/native.h
deleted file mode 100644
index 6d38d01..0000000
--- a/ext/kenlm/jam-files/engine/native.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/* Copyright 2003. David Abrahams
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#ifndef NATIVE_H_VP_2003_12_09
-#define NATIVE_H_VP_2003_12_09
-
-#include "function.h"
-#include "frames.h"
-#include "lists.h"
-#include "object.h"
-
-typedef struct native_rule_t
-{
- OBJECT * name;
- FUNCTION * procedure;
-
- /* Version of the interface that the native rule provides. It is possible
- * that we want to change the set parameter for existing native rule. In
- * that case, version number should be incremented so Boost.Build can check
- * for the version it relies on.
- *
- * Versions are numbered from 1.
- */
- int version;
-} native_rule_t;
-/* MSVC debugger gets confused unless the native_rule_t typedef is provided. */
-
-void declare_native_rule( char const * module, char const * rule,
- char const * * args, LIST * (*f)( FRAME *, int ), int version );
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/object.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/object.c b/ext/kenlm/jam-files/engine/object.c
deleted file mode 100644
index ef46e4a..0000000
--- a/ext/kenlm/jam-files/engine/object.c
+++ /dev/null
@@ -1,394 +0,0 @@
-/*
- * Copyright 1993, 1995 Christopher Seiwald.
- * Copyright 2011 Steven Watanabe
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/*
- * object.c - object manipulation routines
- *
- * External functions:
- * object_new() - create an object from a string
- * object_new_range() - create an object from a string of given length
- * object_copy() - return a copy of an object
- * object_free() - free an object
- * object_str() - get the string value of an object
- * object_done() - free string tables
- *
- * This implementation builds a hash table of all strings, so that multiple
- * calls of object_new() on the same string allocate memory for the string once.
- * Strings are never actually freed.
- */
-
-#include "jam.h"
-#include "object.h"
-
-#include <assert.h>
-#include <stddef.h>
-#include <stdlib.h>
-
-
-#define OBJECT_MAGIC 0xa762e0e3u
-
-#ifndef object_copy
-
-struct hash_header
-{
-#ifndef NDEBUG
- unsigned int magic;
-#endif
- unsigned int hash;
- struct hash_item * next;
-};
-
-#endif
-
-struct hash_item
-{
- struct hash_header header;
- char data[ 1 ];
-};
-
-#define ALLOC_ALIGNMENT (sizeof(struct hash_item) - sizeof(struct hash_header))
-
-typedef struct string_set
-{
- unsigned int num;
- unsigned int size;
- struct hash_item * * data;
-} string_set;
-
-static string_set strhash;
-static int strtotal = 0;
-static int strcount_in = 0;
-static int strcount_out = 0;
-
-
-/*
- * Immortal string allocator implementation speeds string allocation and cuts
- * down on internal fragmentation.
- */
-
-#define STRING_BLOCK 4096
-typedef struct strblock
-{
- struct strblock * next;
- char data[ STRING_BLOCK ];
-} strblock;
-
-static strblock * strblock_chain = 0;
-
-/* Storage remaining in the current strblock */
-static char * storage_start = 0;
-static char * storage_finish = 0;
-
-
-/*
- * allocate() - Allocate n bytes of immortal string storage.
- */
-
-static char * allocate( size_t n )
-{
-#ifdef BJAM_NEWSTR_NO_ALLOCATE
- return (char *)BJAM_MALLOC( n );
-#else
- /* See if we can grab storage from an existing block. */
- size_t remaining = storage_finish - storage_start;
- n = ( ( n + ALLOC_ALIGNMENT - 1 ) / ALLOC_ALIGNMENT ) * ALLOC_ALIGNMENT;
- if ( remaining >= n )
- {
- char * result = storage_start;
- storage_start += n;
- return result;
- }
- else /* Must allocate a new block. */
- {
- strblock * new_block;
- size_t nalloc = n;
- if ( nalloc < STRING_BLOCK )
- nalloc = STRING_BLOCK;
-
- /* Allocate a new block and link into the chain. */
- new_block = (strblock *)BJAM_MALLOC( offsetof( strblock, data[ 0 ] ) +
- nalloc * sizeof( new_block->data[ 0 ] ) );
- if ( new_block == 0 )
- return 0;
- new_block->next = strblock_chain;
- strblock_chain = new_block;
-
- /* Take future allocations out of the larger remaining space. */
- if ( remaining < nalloc - n )
- {
- storage_start = new_block->data + n;
- storage_finish = new_block->data + nalloc;
- }
- return new_block->data;
- }
-#endif
-}
-
-
-static unsigned int hash_keyval( char const * key, int const size )
-{
- unsigned int const magic = 2147059363;
- unsigned int hash = 0;
-
- unsigned int i;
- for ( i = 0; i < size / sizeof( unsigned int ); ++i )
- {
- unsigned int val;
- memcpy( &val, key, sizeof( unsigned int ) );
- hash = hash * magic + val;
- key += sizeof( unsigned int );
- }
-
- {
- unsigned int val = 0;
- memcpy( &val, key, size % sizeof( unsigned int ) );
- hash = hash * magic + val;
- }
-
- return hash + ( hash >> 17 );
-}
-
-
-static void string_set_init( string_set * set )
-{
- set->size = 0;
- set->num = 4;
- set->data = (struct hash_item * *)BJAM_MALLOC( set->num * sizeof( struct hash_item * ) );
- memset( set->data, 0, set->num * sizeof( struct hash_item * ) );
-}
-
-
-static void string_set_done( string_set * set )
-{
- BJAM_FREE( set->data );
-}
-
-
-static void string_set_resize( string_set * set )
-{
- unsigned i;
- string_set new_set;
- new_set.num = set->num * 2;
- new_set.size = set->size;
- new_set.data = (struct hash_item * *)BJAM_MALLOC( sizeof( struct hash_item *
- ) * new_set.num );
- memset( new_set.data, 0, sizeof( struct hash_item * ) * new_set.num );
- for ( i = 0; i < set->num; ++i )
- {
- while ( set->data[ i ] )
- {
- struct hash_item * temp = set->data[ i ];
- unsigned pos = temp->header.hash % new_set.num;
- set->data[ i ] = temp->header.next;
- temp->header.next = new_set.data[ pos ];
- new_set.data[ pos ] = temp;
- }
- }
- BJAM_FREE( set->data );
- *set = new_set;
-}
-
-
-static char const * string_set_insert( string_set * set, char const * string,
- int const size )
-{
- unsigned hash = hash_keyval( string, size );
- unsigned pos = hash % set->num;
-
- struct hash_item * result;
-
- for ( result = set->data[ pos ]; result; result = result->header.next )
- if ( !strncmp( result->data, string, size ) && !result->data[ size ] )
- return result->data;
-
- if ( set->size >= set->num )
- {
- string_set_resize( set );
- pos = hash % set->num;
- }
-
- result = (struct hash_item *)allocate( sizeof( struct hash_header ) + size +
- 1 );
- result->header.hash = hash;
- result->header.next = set->data[ pos ];
-#ifndef NDEBUG
- result->header.magic = OBJECT_MAGIC;
-#endif
- memcpy( result->data, string, size );
- result->data[ size ] = '\0';
- assert( hash_keyval( result->data, size ) == result->header.hash );
- set->data[ pos ] = result;
- strtotal += size + 1;
- ++set->size;
-
- return result->data;
-}
-
-
-static struct hash_item * object_get_item( OBJECT * obj )
-{
- return (struct hash_item *)( (char *)obj - offsetof( struct hash_item, data
- ) );
-}
-
-
-static void object_validate( OBJECT * obj )
-{
- assert( obj );
- assert( object_get_item( obj )->header.magic == OBJECT_MAGIC );
-}
-
-
-/*
- * object_new_range() - create an object from a string of given length
- */
-
-OBJECT * object_new_range( char const * const string, int const size )
-{
- ++strcount_in;
-
-#ifdef BJAM_NO_MEM_CACHE
- {
- struct hash_item * const m = (struct hash_item *)BJAM_MALLOC( sizeof(
- struct hash_header ) + size + 1 );
- strtotal += size + 1;
- memcpy( m->data, string, size );
- m->data[ size ] = '\0';
- m->header.magic = OBJECT_MAGIC;
- return (OBJECT *)m->data;
- }
-#else
- if ( !strhash.data )
- string_set_init( &strhash );
- return (OBJECT *)string_set_insert( &strhash, string, size );
-#endif
-}
-
-
-/*
- * object_new() - create an object from a string
- */
-
-OBJECT * object_new( char const * const string )
-{
- return object_new_range( string, strlen( string ) );
-}
-
-
-#ifndef object_copy
-
-/*
- * object_copy() - return a copy of an object
- */
-
-OBJECT * object_copy( OBJECT * obj )
-{
- object_validate( obj );
-#ifdef BJAM_NO_MEM_CACHE
- return object_new( object_str( obj ) );
-#else
- ++strcount_in;
- return obj;
-#endif
-}
-
-
-/*
- * object_free() - free an object
- */
-
-void object_free( OBJECT * obj )
-{
- object_validate( obj );
-#ifdef BJAM_NO_MEM_CACHE
- BJAM_FREE( object_get_item( obj ) );
-#endif
- ++strcount_out;
-}
-
-
-/*
- * object_str() - return the OBJECT's internal C string
- */
-
-char const * object_str( OBJECT * obj )
-{
- object_validate( obj );
- return (char const *)obj;
-}
-
-
-/*
- * object_equal() - compare two objects
- */
-
-int object_equal( OBJECT * lhs, OBJECT * rhs )
-{
- object_validate( lhs );
- object_validate( rhs );
-#ifdef BJAM_NO_MEM_CACHE
- return !strcmp( object_str( lhs ), object_str( rhs ) );
-#else
- assert( ( lhs == rhs ) == !strcmp( object_str( lhs ), object_str( rhs ) ) );
- return lhs == rhs;
-#endif
-}
-
-
-/*
- * object_hash() - returns the hash value of an object
- */
-
-unsigned int object_hash( OBJECT * obj )
-{
- object_validate( obj );
-#ifdef BJAM_NO_MEM_CACHE
- return hash_keyval( object_str( obj ), strlen( object_str( obj ) ) );
-#else
- return object_get_item( obj )->header.hash;
-#endif
-}
-
-#endif
-
-/*
- * object_done() - free string tables.
- */
-
-void object_done()
-{
-#ifdef BJAM_NEWSTR_NO_ALLOCATE
- unsigned i;
- for ( i = 0; i < strhash.num; ++i )
- {
- while ( strhash.data[ i ] )
- {
- struct hash_item * item = strhash.data[ i ];
- strhash.data[ i ] = item->header.next;
- BJAM_FREE( item );
- }
- }
-#else
- /* Reclaim string blocks. */
- while ( strblock_chain )
- {
- strblock * const n = strblock_chain->next;
- BJAM_FREE( strblock_chain );
- strblock_chain = n;
- }
-#endif
-
- string_set_done( &strhash );
-
- if ( DEBUG_MEM )
- {
- printf( "%dK in strings\n", strtotal / 1024 );
- if ( strcount_in != strcount_out )
- printf( "--- %d strings of %d dangling\n", strcount_in -
- strcount_out, strcount_in );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/object.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/object.h b/ext/kenlm/jam-files/engine/object.h
deleted file mode 100644
index cabb9f6..0000000
--- a/ext/kenlm/jam-files/engine/object.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright 2011 Steven Watanabe
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/*
- * object.h - object manipulation routines
- */
-
-#ifndef BOOST_JAM_OBJECT_H
-#define BOOST_JAM_OBJECT_H
-
-typedef struct _object OBJECT;
-
-OBJECT * object_new( char const * const );
-OBJECT * object_new_range( char const * const, int const size );
-void object_done( void );
-
-#if defined(NDEBUG) && !defined(BJAM_NO_MEM_CACHE)
-
-struct hash_header
-{
- unsigned int hash;
- struct hash_item * next;
-};
-
-#define object_str( obj ) ((char const *)(obj))
-#define object_copy( obj ) (obj)
-#define object_free( obj ) ((void)0)
-#define object_equal( lhs, rhs ) ((lhs) == (rhs))
-#define object_hash( obj ) (((struct hash_header *)((char *)(obj) - sizeof(struct hash_header)))->hash)
-
-#else
-
-char const * object_str ( OBJECT * );
-OBJECT * object_copy ( OBJECT * );
-void object_free ( OBJECT * );
-int object_equal( OBJECT *, OBJECT * );
-unsigned int object_hash ( OBJECT * );
-
-#endif
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/option.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/option.c b/ext/kenlm/jam-files/engine/option.c
deleted file mode 100644
index d25e5e8..0000000
--- a/ext/kenlm/jam-files/engine/option.c
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Copyright 1993, 1995 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-# include "jam.h"
-# include "option.h"
-
-/*
- * option.c - command line option processing
- *
- * {o >o
- * \<>) "Process command line options as defined in <option.h>.
- * Return the number of argv[] elements used up by options,
- * or -1 if an invalid option flag was given or an argument
- * was supplied for an option that does not require one."
- */
-
-int getoptions( int argc, char * * argv, char * opts, bjam_option * optv )
-{
- int i;
- int optc = N_OPTS;
-
- memset( (char *)optv, '\0', sizeof( *optv ) * N_OPTS );
-
- for ( i = 0; i < argc; ++i )
- {
- char *arg;
-
- if ( ( argv[ i ][ 0 ] != '-' ) ||
- ( ( argv[ i ][ 1 ] != '-' ) && !isalpha( argv[ i ][ 1 ] ) ) )
- continue;
-
- if ( !optc-- )
- {
- printf( "too many options (%d max)\n", N_OPTS );
- return -1;
- }
-
- for ( arg = &argv[ i ][ 1 ]; *arg; ++arg )
- {
- char * f;
-
- for ( f = opts; *f; ++f )
- if ( *f == *arg )
- break;
-
- if ( !*f )
- {
- printf( "Invalid option: -%c\n", *arg );
- return -1;
- }
-
- optv->flag = *f;
-
- if ( f[ 1 ] != ':' )
- {
- optv++->val = "true";
- }
- else if ( arg[ 1 ] )
- {
- optv++->val = &arg[1];
- break;
- }
- else if ( ++i < argc )
- {
- optv++->val = argv[ i ];
- break;
- }
- else
- {
- printf( "option: -%c needs argument\n", *f );
- return -1;
- }
- }
- }
-
- return i;
-}
-
-
-/*
- * Name: getoptval() - find an option given its character.
- */
-
-char * getoptval( bjam_option * optv, char opt, int subopt )
-{
- int i;
- for ( i = 0; i < N_OPTS; ++i, ++optv )
- if ( ( optv->flag == opt ) && !subopt-- )
- return optv->val;
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/option.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/option.h b/ext/kenlm/jam-files/engine/option.h
deleted file mode 100644
index 7c9c747..0000000
--- a/ext/kenlm/jam-files/engine/option.h
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright 1993, 1995 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/*
- * option.h - command line option processing
- *
- * {o >o
- * \ -) "Command line option."
- */
-
-typedef struct bjam_option
-{
- char flag; /* filled in by getoption() */
- char * val; /* set to random address if true */
-} bjam_option;
-
-#define N_OPTS 256
-
-int getoptions( int argc, char * * argv, char * opts, bjam_option * optv );
-char * getoptval( bjam_option * optv, char opt, int subopt );
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/output.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/output.c b/ext/kenlm/jam-files/engine/output.c
deleted file mode 100644
index eaaee43..0000000
--- a/ext/kenlm/jam-files/engine/output.c
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- Copyright 2007 Rene Rivera
- Distributed under the Boost Software License, Version 1.0.
- (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-*/
-
-#include "jam.h"
-#include "output.h"
-
-#include <stdio.h>
-
-
-#define bjam_out (stdout)
-#define bjam_err (stderr)
-
-static void out_( char const * data, FILE * const io )
-{
- while ( *data )
- {
- size_t const len = strcspn( data, "\r" );
- data += fwrite( data, 1, len, io );
- if ( *data == '\r' ) ++data;
- }
-}
-
-
-void out_action
-(
- char const * const action,
- char const * const target,
- char const * const command,
- char const * const out_data,
- char const * const err_data,
- int const exit_reason
-)
-{
- /* Print out the action + target line, if the action is quiet the action
- * should be null.
- */
- if ( action )
- fprintf( bjam_out, "%s %s\n", action, target );
-
- /* Print out the command executed if given -d+2. */
- if ( DEBUG_EXEC )
- {
- fputs( command, bjam_out );
- fputc( '\n', bjam_out );
- }
-
- /* Print out the command executed to the command stream. */
- if ( globs.cmdout )
- fputs( command, globs.cmdout );
-
- /* If the process expired, make user aware with an explicit message, but do
- * this only for non-quiet actions.
- */
- if ( exit_reason == EXIT_TIMEOUT && action )
- fprintf( bjam_out, "%ld second time limit exceeded\n", globs.timeout );
-
- /* Print out the command output, if requested, or if the program failed, but
- * only output for non-quiet actions.
- */
- if ( action || exit_reason != EXIT_OK )
- {
- if ( out_data &&
- ( ( globs.pipe_action & 1 /* STDOUT_FILENO */ ) ||
- ( globs.pipe_action == 0 ) ) )
- out_( out_data, bjam_out );
- if ( err_data && ( globs.pipe_action & 2 /* STDERR_FILENO */ ) )
- out_( err_data, bjam_err );
- }
-
- fflush( bjam_out );
- fflush( bjam_err );
- fflush( globs.cmdout );
-}
-
-
-OBJECT * outf_int( int const value )
-{
- char buffer[ 50 ];
- sprintf( buffer, "%i", value );
- return object_new( buffer );
-}
-
-
-OBJECT * outf_double( double const value )
-{
- char buffer[ 50 ];
- sprintf( buffer, "%f", value );
- return object_new( buffer );
-}
-
-
-OBJECT * outf_time( timestamp const * const time )
-{
- return object_new( timestamp_str( time ) );
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/output.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/output.h b/ext/kenlm/jam-files/engine/output.h
deleted file mode 100644
index 186e867..0000000
--- a/ext/kenlm/jam-files/engine/output.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- Copyright 2007 Rene Rivera
- Distributed under the Boost Software License, Version 1.0.
- (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-*/
-
-#ifndef BJAM_OUTPUT_H
-#define BJAM_OUTPUT_H
-
-#include "object.h"
-#include "timestamp.h"
-
-#define EXIT_OK 0
-#define EXIT_FAIL 1
-#define EXIT_TIMEOUT 2
-
-void out_action(
- char const * const action,
- char const * const target,
- char const * const command,
- char const * const out_data,
- char const * const err_data,
- int const exit_reason
-);
-
-OBJECT * outf_int( int const value );
-OBJECT * outf_double( double const value );
-OBJECT * outf_time( timestamp const * const value );
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/parse.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/parse.c b/ext/kenlm/jam-files/engine/parse.c
deleted file mode 100644
index 02412e0..0000000
--- a/ext/kenlm/jam-files/engine/parse.c
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Copyright 1993, 2000 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#include "jam.h"
-#include "lists.h"
-#include "parse.h"
-#include "scan.h"
-#include "object.h"
-#include "modules.h"
-#include "frames.h"
-#include "function.h"
-
-/*
- * parse.c - make and destroy parse trees as driven by the parser
- *
- * 09/07/00 (seiwald) - ref count on PARSE to avoid freeing when used,
- * as per Matt Armstrong.
- * 09/11/00 (seiwald) - structure reworked to reflect that (*func)()
- * returns a LIST *.
- */
-
-static PARSE * yypsave;
-
-void parse_file( OBJECT * f, FRAME * frame )
-{
- /* Suspend scan of current file and push this new file in the stream. */
- yyfparse( f );
-
- /* Now parse each block of rules and execute it. Execute it outside of the
- * parser so that recursive calls to yyrun() work (no recursive yyparse's).
- */
-
- for ( ; ; )
- {
- PARSE * p;
- FUNCTION * func;
-
- /* Filled by yyparse() calling parse_save(). */
- yypsave = 0;
-
- /* If parse error or empty parse, outta here. */
- if ( yyparse() || !( p = yypsave ) )
- break;
-
- /* Run the parse tree. */
- func = function_compile( p );
- parse_free( p );
- list_free( function_run( func, frame, stack_global() ) );
- function_free( func );
- }
-}
-
-
-void parse_save( PARSE * p )
-{
- yypsave = p;
-}
-
-
-PARSE * parse_make(
- int type,
- PARSE * left,
- PARSE * right,
- PARSE * third,
- OBJECT * string,
- OBJECT * string1,
- int num )
-{
- PARSE * p = (PARSE *)BJAM_MALLOC( sizeof( PARSE ) );
-
- p->type = type;
- p->left = left;
- p->right = right;
- p->third = third;
- p->string = string;
- p->string1 = string1;
- p->num = num;
- p->refs = 1;
- p->rulename = 0;
-
- if ( left )
- {
- p->file = object_copy( left->file );
- p->line = left->line;
- }
- else
- {
- yyinput_last_read_token( &p->file, &p->line );
- p->file = object_copy( p->file );
- }
-
- return p;
-}
-
-
-void parse_refer( PARSE * p )
-{
- ++p->refs;
-}
-
-
-void parse_free( PARSE * p )
-{
- if ( --p->refs )
- return;
-
- if ( p->string )
- object_free( p->string );
- if ( p->string1 )
- object_free( p->string1 );
- if ( p->left )
- parse_free( p->left );
- if ( p->right )
- parse_free( p->right );
- if ( p->third )
- parse_free( p->third );
- if ( p->rulename )
- object_free( p->rulename );
- if ( p->file )
- object_free( p->file );
-
- BJAM_FREE( (char *)p );
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/parse.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/parse.h b/ext/kenlm/jam-files/engine/parse.h
deleted file mode 100644
index bb47af6..0000000
--- a/ext/kenlm/jam-files/engine/parse.h
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Copyright 1993, 2000 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * parse.h - make and destroy parse trees as driven by the parser.
- */
-
-#ifndef PARSE_DWA20011020_H
-#define PARSE_DWA20011020_H
-
-#include "frames.h"
-#include "lists.h"
-#include "modules.h"
-
-
-#define PARSE_APPEND 0
-#define PARSE_FOREACH 1
-#define PARSE_IF 2
-#define PARSE_EVAL 3
-#define PARSE_INCLUDE 4
-#define PARSE_LIST 5
-#define PARSE_LOCAL 6
-#define PARSE_MODULE 7
-#define PARSE_CLASS 8
-#define PARSE_NULL 9
-#define PARSE_ON 10
-#define PARSE_RULE 11
-#define PARSE_RULES 12
-#define PARSE_SET 13
-#define PARSE_SETCOMP 14
-#define PARSE_SETEXEC 15
-#define PARSE_SETTINGS 16
-#define PARSE_SWITCH 17
-#define PARSE_WHILE 18
-
-
-/*
- * Parse tree node.
- */
-
-typedef struct _PARSE PARSE;
-
-struct _PARSE {
- int type;
- PARSE * left;
- PARSE * right;
- PARSE * third;
- OBJECT * string;
- OBJECT * string1;
- int num;
- int refs;
- OBJECT * rulename;
- OBJECT * file;
- int line;
-};
-
-void parse_file( OBJECT *, FRAME * );
-void parse_save( PARSE * );
-
-PARSE * parse_make( int type, PARSE * left, PARSE * right, PARSE * third,
- OBJECT * string, OBJECT * string1, int num );
-
-void parse_refer( PARSE * );
-void parse_free( PARSE * );
-LIST * parse_evaluate( PARSE *, FRAME * );
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/patchlevel.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/patchlevel.h b/ext/kenlm/jam-files/engine/patchlevel.h
deleted file mode 100644
index 60b0d61..0000000
--- a/ext/kenlm/jam-files/engine/patchlevel.h
+++ /dev/null
@@ -1,17 +0,0 @@
-/*
- * Copyright 1993-2002 Christopher Seiwald and Perforce Software, Inc.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* Keep JAMVERSYM in sync with VERSION. */
-/* It can be accessed as $(JAMVERSION) in the Jamfile. */
-
-#define VERSION_MAJOR 2011
-#define VERSION_MINOR 12
-#define VERSION_PATCH 1
-#define VERSION_MAJOR_SYM "2011"
-#define VERSION_MINOR_SYM "12"
-#define VERSION_PATCH_SYM "01"
-#define VERSION "2011.12.1"
-#define JAMVERSYM "JAMVERSION=2011.12"
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/pathnt.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/pathnt.c b/ext/kenlm/jam-files/engine/pathnt.c
deleted file mode 100644
index 8abf8fe..0000000
--- a/ext/kenlm/jam-files/engine/pathnt.c
+++ /dev/null
@@ -1,301 +0,0 @@
-/*
- * Copyright 1993-2002 Christopher Seiwald and Perforce Software, Inc.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Copyright 2005 Rene Rivera.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * pathnt.c - NT specific path manipulation support
- */
-
-#include "pathsys.h"
-
-#include "hash.h"
-
-#define WIN32_LEAN_AND_MEAN
-#include <windows.h>
-
-#include <assert.h>
-#include <stdlib.h>
-
-
-/* The definition of this in winnt.h is not ANSI-C compatible. */
-#undef INVALID_FILE_ATTRIBUTES
-#define INVALID_FILE_ATTRIBUTES ((DWORD)-1)
-
-
-typedef struct path_key_entry
-{
- OBJECT * path;
- OBJECT * key;
- int exists;
-} path_key_entry;
-
-static struct hash * path_key_cache;
-
-
-/*
- * path_get_process_id_()
- */
-
-unsigned long path_get_process_id_( void )
-{
- return GetCurrentProcessId();
-}
-
-
-/*
- * path_get_temp_path_()
- */
-
-void path_get_temp_path_( string * buffer )
-{
- DWORD pathLength = GetTempPathA( 0, NULL );
- string_reserve( buffer, pathLength );
- pathLength = GetTempPathA( pathLength, buffer->value );
- buffer->value[ pathLength - 1 ] = '\0';
- buffer->size = pathLength - 1;
-}
-
-
-/*
- * canonicWindowsPath() - convert a given path into its canonic/long format
- *
- * Appends the canonic path to the end of the given 'string' object.
- *
- * FIXME: This function is still work-in-progress as it originally did not
- * necessarily return the canonic path format (could return slightly different
- * results for certain equivalent path strings) and could accept paths pointing
- * to non-existing file system entities as well.
- *
- * Caches results internally, automatically caching any parent paths it has to
- * convert to their canonic format in the process.
- *
- * Prerequisites:
- * - path given in normalized form, i.e. all of its folder separators have
- * already been converted into '\\'
- * - path_key_cache path/key mapping cache object already initialized
- */
-
-static int canonicWindowsPath( char const * const path, int const path_length,
- string * const out )
-{
- char const * last_element;
- unsigned long saved_size;
- char const * p;
- int missing_parent;
-
- /* This is only called via path_key(), which initializes the cache. */
- assert( path_key_cache );
-
- if ( !path_length )
- return 1;
-
- if ( path_length == 1 && path[ 0 ] == '\\' )
- {
- string_push_back( out, '\\' );
- return 1;
- }
-
- if ( path[ 1 ] == ':' &&
- ( path_length == 2 ||
- ( path_length == 3 && path[ 2 ] == '\\' ) ) )
- {
- string_push_back( out, toupper( path[ 0 ] ) );
- string_push_back( out, ':' );
- string_push_back( out, '\\' );
- return 1;
- }
-
- /* Find last '\\'. */
- for ( p = path + path_length - 1; p >= path && *p != '\\'; --p );
- last_element = p + 1;
-
- /* Special case '\' && 'D:\' - include trailing '\'. */
- if ( p == path ||
- p == path + 2 && path[ 1 ] == ':' )
- ++p;
-
- missing_parent = 0;
-
- if ( p >= path )
- {
- char const * const dir = path;
- int const dir_length = p - path;
- OBJECT * const dir_obj = object_new_range( dir, dir_length );
- int found;
- path_key_entry * const result = (path_key_entry *)hash_insert(
- path_key_cache, dir_obj, &found );
- if ( !found )
- {
- result->path = dir_obj;
- if ( canonicWindowsPath( dir, dir_length, out ) )
- result->exists = 1;
- else
- result->exists = 0;
- result->key = object_new( out->value );
- }
- else
- {
- object_free( dir_obj );
- string_append( out, object_str( result->key ) );
- }
- if ( !result->exists )
- missing_parent = 1;
- }
-
- if ( out->size && out->value[ out->size - 1 ] != '\\' )
- string_push_back( out, '\\' );
-
- saved_size = out->size;
- string_append_range( out, last_element, path + path_length );
-
- if ( !missing_parent )
- {
- char const * const n = last_element;
- int const n_length = path + path_length - n;
- if ( !( n_length == 1 && n[ 0 ] == '.' )
- && !( n_length == 2 && n[ 0 ] == '.' && n[ 1 ] == '.' ) )
- {
- WIN32_FIND_DATA fd;
- HANDLE const hf = FindFirstFileA( out->value, &fd );
- if ( hf != INVALID_HANDLE_VALUE )
- {
- string_truncate( out, saved_size );
- string_append( out, fd.cFileName );
- FindClose( hf );
- return 1;
- }
- }
- }
- return 0;
-}
-
-
-/*
- * normalize_path() - 'normalizes' the given path for the path-key mapping
- *
- * The resulting string has nothing to do with 'normalized paths' as used in
- * Boost Jam build scripts and the built-in NORMALIZE_PATH rule. It is intended
- * to be used solely as an intermediate step when mapping an arbitrary path to
- * its canonical representation.
- *
- * When choosing the intermediate string the important things are for it to be
- * inexpensive to calculate and any two paths having different canonical
- * representations also need to have different calculated intermediate string
- * representations. Any implemented additional rules serve only to simplify
- * constructing the canonical path representation from the calculated
- * intermediate string.
- *
- * Implemented returned path rules:
- * - use backslashes as path separators
- * - lowercase only (since all Windows file systems are case insensitive)
- * - trim trailing path separator except in case of a root path, i.e. 'X:\'
- */
-
-static void normalize_path( string * path )
-{
- char * s;
- for ( s = path->value; s < path->value + path->size; ++s )
- *s = *s == '/' ? '\\' : tolower( *s );
- /* Strip trailing "/". */
- if ( path->size && path->size != 3 && path->value[ path->size - 1 ] == '\\'
- )
- string_pop_back( path );
-}
-
-
-static path_key_entry * path_key( OBJECT * const path,
- int const known_to_be_canonic )
-{
- path_key_entry * result;
- int found;
-
- if ( !path_key_cache )
- path_key_cache = hashinit( sizeof( path_key_entry ), "path to key" );
-
- result = (path_key_entry *)hash_insert( path_key_cache, path, &found );
- if ( !found )
- {
- OBJECT * normalized;
- int normalized_size;
- path_key_entry * nresult;
- result->path = path;
- {
- string buf[ 1 ];
- string_copy( buf, object_str( path ) );
- normalize_path( buf );
- normalized = object_new( buf->value );
- normalized_size = buf->size;
- string_free( buf );
- }
- nresult = (path_key_entry *)hash_insert( path_key_cache, normalized,
- &found );
- if ( !found || nresult == result )
- {
- nresult->path = normalized;
- if ( known_to_be_canonic )
- nresult->key = object_copy( path );
- else
- {
- string canonic_path[ 1 ];
- string_new( canonic_path );
- if ( canonicWindowsPath( object_str( normalized ), normalized_size,
- canonic_path ) )
- nresult->exists = 1;
- else
- nresult->exists = 0;
- nresult->key = object_new( canonic_path->value );
- string_free( canonic_path );
- }
- }
- else
- object_free( normalized );
- if ( nresult != result )
- {
- result->path = object_copy( path );
- result->key = object_copy( nresult->key );
- result->exists = nresult->exists;
- }
- }
-
- return result;
-}
-
-
-void path_register_key( OBJECT * canonic_path )
-{
- path_key( canonic_path, 1 );
-}
-
-
-OBJECT * path_as_key( OBJECT * path )
-{
- return object_copy( path_key( path, 0 )->key );
-}
-
-
-static void free_path_key_entry( void * xentry, void * const data )
-{
- path_key_entry * const entry = (path_key_entry *)xentry;
- object_free( entry->path );
- object_free( entry->key );
-}
-
-
-void path_done( void )
-{
- if ( path_key_cache )
- {
- hashenumerate( path_key_cache, &free_path_key_entry, 0 );
- hashdone( path_key_cache );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/pathsys.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/pathsys.c b/ext/kenlm/jam-files/engine/pathsys.c
deleted file mode 100644
index ae4e6e0..0000000
--- a/ext/kenlm/jam-files/engine/pathsys.c
+++ /dev/null
@@ -1,285 +0,0 @@
-/*
- * Copyright 1993-2002 Christopher Seiwald and Perforce Software, Inc.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Copyright 2005 Rene Rivera.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * pathsys.c - platform independent path manipulation support
- *
- * External routines:
- * path_build() - build a filename given dir/base/suffix/member
- * path_parent() - make a PATHNAME point to its parent dir
- * path_parse() - split a file name into dir/base/suffix/member
- * path_tmpdir() - returns the system dependent temporary folder path
- * path_tmpfile() - returns a new temporary path
- * path_tmpnam() - returns a new temporary name
- *
- * File_parse() and path_build() just manipulate a string and a structure;
- * they do not make system calls.
- */
-
-#include "jam.h"
-#include "pathsys.h"
-
-#include "filesys.h"
-
-#include <stdlib.h>
-#include <time.h>
-
-
-/* Internal OS specific implementation details - have names ending with an
- * underscore and are expected to be implemented in an OS specific pathXXX.c
- * module.
- */
-unsigned long path_get_process_id_( void );
-void path_get_temp_path_( string * buffer );
-
-
-/*
- * path_parse() - split a file name into dir/base/suffix/member
- */
-
-void path_parse( char const * file, PATHNAME * f )
-{
- char const * p;
- char const * q;
- char const * end;
-
- memset( (char *)f, 0, sizeof( *f ) );
-
- /* Look for '<grist>'. */
-
- if ( ( file[ 0 ] == '<' ) && ( p = strchr( file, '>' ) ) )
- {
- f->f_grist.ptr = file;
- f->f_grist.len = p - file;
- file = p + 1;
- }
-
- /* Look for 'dir/'. */
-
- p = strrchr( file, '/' );
-
-#if PATH_DELIM == '\\'
- /* On NT, look for dir\ as well */
- {
- char * const p1 = strrchr( p ? p + 1 : file, '\\' );
- if ( p1 ) p = p1;
- }
-#endif
-
- if ( p )
- {
- f->f_dir.ptr = file;
- f->f_dir.len = p - file;
-
- /* Special case for / - dirname is /, not "" */
- if ( !f->f_dir.len )
- ++f->f_dir.len;
-
-#if PATH_DELIM == '\\'
- /* Special case for D:/ - dirname is D:/, not "D:" */
- if ( f->f_dir.len == 2 && file[ 1 ] == ':' )
- ++f->f_dir.len;
-#endif
-
- file = p + 1;
- }
-
- end = file + strlen( file );
-
- /* Look for '(member)'. */
- if ( ( p = strchr( file, '(' ) ) && ( end[ -1 ] == ')' ) )
- {
- f->f_member.ptr = p + 1;
- f->f_member.len = end - p - 2;
- end = p;
- }
-
- /* Look for '.suffix'. This would be memrchr(). */
- p = 0;
- for ( q = file; ( q = (char *)memchr( q, '.', end - q ) ); ++q )
- p = q;
- if ( p )
- {
- f->f_suffix.ptr = p;
- f->f_suffix.len = end - p;
- end = p;
- }
-
- /* Leaves base. */
- f->f_base.ptr = file;
- f->f_base.len = end - file;
-}
-
-
-/*
- * is_path_delim() - true iff c is a path delimiter
- */
-
-static int is_path_delim( char const c )
-{
- return c == PATH_DELIM
-#if PATH_DELIM == '\\'
- || c == '/'
-#endif
- ;
-}
-
-
-/*
- * as_path_delim() - convert c to a path delimiter if it is not one already
- */
-
-static char as_path_delim( char const c )
-{
- return is_path_delim( c ) ? c : PATH_DELIM;
-}
-
-
-/*
- * path_build() - build a filename given dir/base/suffix/member
- *
- * To avoid changing slash direction on NT when reconstituting paths, instead of
- * unconditionally appending PATH_DELIM we check the past-the-end character of
- * the previous path element. If it is a path delimiter, we append that, and
- * only append PATH_DELIM as a last resort. This heuristic is based on the fact
- * that PATHNAME objects are usually the result of calling path_parse, which
- * leaves the original slashes in the past-the-end position. Correctness depends
- * on the assumption that all strings are zero terminated, so a past-the-end
- * character will always be available.
- *
- * As an attendant patch, we had to ensure that backslashes are used explicitly
- * in 'timestamp.c'.
- */
-
-void path_build( PATHNAME * f, string * file )
-{
- file_build1( f, file );
-
- /* Do not prepend root if it is '.' or the directory is rooted. */
- if ( f->f_root.len
- && !( f->f_root.len == 1 && f->f_root.ptr[ 0 ] == '.' )
- && !( f->f_dir.len && f->f_dir.ptr[ 0 ] == '/' )
-#if PATH_DELIM == '\\'
- && !( f->f_dir.len && f->f_dir.ptr[ 0 ] == '\\' )
- && !( f->f_dir.len && f->f_dir.ptr[ 1 ] == ':' )
-#endif
- )
- {
- string_append_range( file, f->f_root.ptr, f->f_root.ptr + f->f_root.len
- );
- /* If 'root' already ends with a path delimeter, do not add another one.
- */
- if ( !is_path_delim( f->f_root.ptr[ f->f_root.len - 1 ] ) )
- string_push_back( file, as_path_delim( f->f_root.ptr[ f->f_root.len
- ] ) );
- }
-
- if ( f->f_dir.len )
- string_append_range( file, f->f_dir.ptr, f->f_dir.ptr + f->f_dir.len );
-
- /* Put path separator between dir and file. */
- /* Special case for root dir: do not add another path separator. */
- if ( f->f_dir.len && ( f->f_base.len || f->f_suffix.len )
-#if PATH_DELIM == '\\'
- && !( f->f_dir.len == 3 && f->f_dir.ptr[ 1 ] == ':' )
-#endif
- && !( f->f_dir.len == 1 && is_path_delim( f->f_dir.ptr[ 0 ] ) ) )
- string_push_back( file, as_path_delim( f->f_dir.ptr[ f->f_dir.len ] ) );
-
- if ( f->f_base.len )
- string_append_range( file, f->f_base.ptr, f->f_base.ptr + f->f_base.len
- );
-
- if ( f->f_suffix.len )
- string_append_range( file, f->f_suffix.ptr, f->f_suffix.ptr +
- f->f_suffix.len );
-
- if ( f->f_member.len )
- {
- string_push_back( file, '(' );
- string_append_range( file, f->f_member.ptr, f->f_member.ptr +
- f->f_member.len );
- string_push_back( file, ')' );
- }
-}
-
-
-/*
- * path_parent() - make a PATHNAME point to its parent dir
- */
-
-void path_parent( PATHNAME * f )
-{
- f->f_base.ptr = f->f_suffix.ptr = f->f_member.ptr = "";
- f->f_base.len = f->f_suffix.len = f->f_member.len = 0;
-}
-
-
-/*
- * path_tmpdir() - returns the system dependent temporary folder path
- *
- * Returned value is stored inside a static buffer and should not be modified.
- * Returned value does *not* include a trailing path separator.
- */
-
-string const * path_tmpdir()
-{
- static string buffer[ 1 ];
- static int have_result;
- if ( !have_result )
- {
- string_new( buffer );
- path_get_temp_path_( buffer );
- have_result = 1;
- }
- return buffer;
-}
-
-
-/*
- * path_tmpnam() - returns a new temporary name
- */
-
-OBJECT * path_tmpnam( void )
-{
- char name_buffer[ 64 ];
- unsigned long const pid = path_get_process_id_();
- static unsigned long t;
- if ( !t ) t = time( 0 ) & 0xffff;
- t += 1;
- sprintf( name_buffer, "jam%lx%lx.000", pid, t );
- return object_new( name_buffer );
-}
-
-
-/*
- * path_tmpfile() - returns a new temporary path
- */
-
-OBJECT * path_tmpfile( void )
-{
- OBJECT * result;
- OBJECT * tmpnam;
-
- string file_path[ 1 ];
- string_copy( file_path, path_tmpdir()->value );
- string_push_back( file_path, PATH_DELIM );
- tmpnam = path_tmpnam();
- string_append( file_path, object_str( tmpnam ) );
- object_free( tmpnam );
- result = object_new( file_path->value );
- string_free( file_path );
-
- return result;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/pathsys.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/pathsys.h b/ext/kenlm/jam-files/engine/pathsys.h
deleted file mode 100644
index 9b7a4ca..0000000
--- a/ext/kenlm/jam-files/engine/pathsys.h
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Copyright 1993-2002 Christopher Seiwald and Perforce Software, Inc.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/*
- * pathsys.h - PATHNAME struct
- */
-
-/*
- * PATHNAME - a name of a file, broken into <grist>dir/base/suffix(member)
- *
- * <grist> - salt to distinguish between targets that would otherwise have the
- * same name - it never appears in the bound name of a target.
- *
- * (member) - archive member name: the syntax is arbitrary, but must agree in
- * path_parse(), path_build() and the Jambase.
- */
-
-#ifndef PATHSYS_VP_20020211_H
-#define PATHSYS_VP_20020211_H
-
-#include "object.h"
-#include "strings.h"
-
-
-typedef struct _pathpart
-{
- char const * ptr;
- int len;
-} PATHPART;
-
-typedef struct _pathname
-{
- PATHPART part[ 6 ];
-
-#define f_grist part[ 0 ]
-#define f_root part[ 1 ]
-#define f_dir part[ 2 ]
-#define f_base part[ 3 ]
-#define f_suffix part[ 4 ]
-#define f_member part[ 5 ]
-} PATHNAME;
-
-
-void path_build( PATHNAME *, string * file );
-void path_parse( char const * file, PATHNAME * );
-void path_parent( PATHNAME * );
-
-/* Given a path, returns an object containing an equivalent path in canonical
- * format that can be used as a unique key for that path. Equivalent paths such
- * as a/b, A\B, and a\B on NT all yield the same key.
- */
-OBJECT * path_as_key( OBJECT * path );
-
-/* Called as an optimization when we know we have a path that is already in its
- * canonical/long/key form. Avoids the need for some subsequent path_as_key()
- * call to do a potentially expensive path conversion requiring access to the
- * actual underlying file system.
- */
-void path_register_key( OBJECT * canonic_path );
-
-/* Returns a static pointer to the system dependent path to the temporary
- * directory. NOTE: Does *not* include a trailing path separator.
- */
-string const * path_tmpdir( void );
-
-/* Returns a new temporary name. */
-OBJECT * path_tmpnam( void );
-
-/* Returns a new temporary path. */
-OBJECT * path_tmpfile( void );
-
-/* Give the first argument to 'main', return a full path to our executable.
- * Returns null in the unlikely case it cannot be determined. Caller is
- * responsible for freeing the string.
- *
- * Implemented in jam.c
- */
-char * executable_path( char const * argv0 );
-
-void path_done( void );
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/pathunix.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/pathunix.c b/ext/kenlm/jam-files/engine/pathunix.c
deleted file mode 100644
index 8ca0d18..0000000
--- a/ext/kenlm/jam-files/engine/pathunix.c
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Copyright 1993-2002 Christopher Seiwald and Perforce Software, Inc.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Copyright 2005 Rene Rivera.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * pathunix.c - UNIX specific path manipulation support
- */
-
-#include "pathsys.h"
-
-#include <stdlib.h>
-#include <unistd.h> /* needed for getpid() */
-
-
-/*
- * path_get_process_id_()
- */
-
-unsigned long path_get_process_id_( void )
-{
- return getpid();
-}
-
-
-/*
- * path_get_temp_path_()
- */
-
-void path_get_temp_path_( string * buffer )
-{
- char const * t = getenv( "TMPDIR" );
- string_append( buffer, t ? t : "/tmp" );
-}
-
-
-/*
- * path_register_key()
- */
-
-void path_register_key( OBJECT * path )
-{
-}
-
-
-/*
- * path_as_key()
- */
-
-OBJECT * path_as_key( OBJECT * path )
-{
- return object_copy( path );
-}
-
-
-/*
- * path_done()
- */
-
-void path_done( void )
-{
-}
[05/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/python/kenlm.pyx
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/python/kenlm.pyx b/ext/kenlm/python/kenlm.pyx
deleted file mode 100644
index f312c90..0000000
--- a/ext/kenlm/python/kenlm.pyx
+++ /dev/null
@@ -1,231 +0,0 @@
-import os
-cimport _kenlm
-
-cdef bytes as_str(data):
- if isinstance(data, bytes):
- return data
- elif isinstance(data, unicode):
- return data.encode('utf8')
- raise TypeError('Cannot convert %s to string' % type(data))
-
-cdef class FullScoreReturn:
- """
- Wrapper around FullScoreReturn.
-
- Notes:
- `prob` has been renamed to `log_prob`
- `oov` has been added to flag whether the word is OOV
- """
-
- cdef float log_prob
- cdef int ngram_length
- cdef bint oov
-
- def __cinit__(self, log_prob, ngram_length, oov):
- self.log_prob = log_prob
- self.ngram_length = ngram_length
- self.oov = oov
-
- def __repr__(self):
- return '{0}({1}, {2}, {3})'.format(self.__class__.__name__, repr(self.log_prob), repr(self.ngram_length), repr(self.oov))
-
- property log_prob:
- def __get__(self):
- return self.log_prob
-
- property ngram_length:
- def __get__(self):
- return self.ngram_length
-
- property oov:
- def __get__(self):
- return self.oov
-
-cdef class State:
- """
- Wrapper around lm::ngram::State so that python code can make incremental queries.
-
- Notes:
- * rich comparisons
- * hashable
- """
-
- cdef _kenlm.State _c_state
-
- def __richcmp__(State qa, State qb, int op):
- r = qa._c_state.Compare(qb._c_state)
- if op == 0: # <
- return r < 0
- elif op == 1: # <=
- return r <= 0
- elif op == 2: # ==
- return r == 0
- elif op == 3: # !=
- return r != 0
- elif op == 4: # >
- return r > 0
- else: # >=
- return r >= 0
-
- def __hash__(self):
- return _kenlm.hash_value(self._c_state)
-
-
-cdef class LanguageModel:
- """
- This is not a strict wrapper, the interface is more pythonic.
- It loads models and queries full sentences.
- """
-
- cdef _kenlm.Model* model
- cdef public bytes path
- cdef _kenlm.const_Vocabulary* vocab
-
- def __init__(self, path):
- """
- Load the language model.
-
- :param path: path to an arpa file or a kenlm binary file.
- """
- self.path = os.path.abspath(as_str(path))
- try:
- self.model = _kenlm.LoadVirtual(self.path)
- except RuntimeError as exception:
- exception_message = str(exception).replace('\n', ' ')
- raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\
- from exception
- self.vocab = &self.model.BaseVocabulary()
-
- def __dealloc__(self):
- del self.model
-
- property order:
- def __get__(self):
- return self.model.Order()
-
- def score(self, sentence, bos = True, eos = True):
- cdef list words = as_str(sentence).split()
- cdef _kenlm.State state
- if bos:
- self.model.BeginSentenceWrite(&state)
- else:
- self.model.NullContextWrite(&state)
- cdef _kenlm.State out_state
- cdef float total = 0
- for word in words:
- total += self.model.BaseScore(&state, self.vocab.Index(word), &out_state)
- state = out_state
- if eos:
- total += self.model.BaseScore(&state, self.vocab.EndSentence(), &out_state)
- return total
-
- def full_scores(self, sentence, bos = True, eos = True):
- """
- full_scores(sentence, bos = True, eos = Ture) -> generate full scores (prob, ngram length, oov)
- @param sentence is a string (do not use boundary symbols)
- @param bos should kenlm add a bos state
- @param eos should kenlm add an eos state
- """
- cdef list words = as_str(sentence).split()
- cdef _kenlm.State state
- if bos:
- self.model.BeginSentenceWrite(&state)
- else:
- self.model.NullContextWrite(&state)
- cdef _kenlm.State out_state
- cdef _kenlm.FullScoreReturn ret
- cdef float total = 0
- cdef _kenlm.WordIndex wid
- for word in words:
- wid = self.vocab.Index(word)
- ret = self.model.BaseFullScore(&state, wid, &out_state)
- yield (ret.prob, ret.ngram_length, wid == 0)
- state = out_state
- if eos:
- ret = self.model.BaseFullScore(&state,
- self.vocab.EndSentence(), &out_state)
- yield (ret.prob, ret.ngram_length, False)
-
- def __contains__(self, word):
- cdef bytes w = as_str(word)
- return (self.vocab.Index(w) != 0)
-
- def __repr__(self):
- return '<LanguageModel from {0}>'.format(os.path.basename(self.path))
-
- def __reduce__(self):
- return (_kenlm.LanguageModel, (self.path,))
-
-cdef class Model:
- """
- This is closer to a wrapper around lm::ngram::Model.
- """
-
- cdef _kenlm.Model* model
- cdef public bytes path
- cdef _kenlm.const_Vocabulary* vocab
-
- def __init__(self, path):
- """
- Load the language model.
-
- :param path: path to an arpa file or a kenlm binary file.
- """
- self.path = os.path.abspath(as_str(path))
- try:
- self.model = _kenlm.LoadVirtual(self.path)
- except RuntimeError as exception:
- exception_message = str(exception).replace('\n', ' ')
- raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\
- from exception
- self.vocab = &self.model.BaseVocabulary()
-
- def __dealloc__(self):
- del self.model
-
- property order:
- def __get__(self):
- return self.model.Order()
-
- def BeginSentenceWrite(self, State state):
- """Change the given state to a BOS state."""
- self.model.BeginSentenceWrite(&state._c_state)
-
- def NullContextWrite(self, State state):
- """Change the given state to a NULL state."""
- self.model.NullContextWrite(&state._c_state)
-
- def BaseScore(self, State in_state, str word, State out_state):
- """
- Return p(word|in_state) and update the output state.
- Wrapper around model.BaseScore(in_state, Index(word), out_state)
-
- :param word: the suffix
- :param state: the context (defaults to NullContext)
- :returns: p(word|state)
- """
- cdef float total = self.model.BaseScore(&in_state._c_state, self.vocab.Index(as_str(word)), &out_state._c_state)
- return total
-
- def BaseFullScore(self, State in_state, str word, State out_state):
- """
- Wrapper around model.BaseScore(in_state, Index(word), out_state)
-
- :param word: the suffix
- :param state: the context (defaults to NullContext)
- :returns: FullScoreReturn(word|state)
- """
- cdef _kenlm.WordIndex wid = self.vocab.Index(as_str(word))
- cdef _kenlm.FullScoreReturn ret = self.model.BaseFullScore(&in_state._c_state, wid, &out_state._c_state)
- return FullScoreReturn(ret.prob, ret.ngram_length, wid == 0)
-
- def __contains__(self, word):
- cdef bytes w = as_str(word)
- return (self.vocab.Index(w) != 0)
-
- def __repr__(self):
- return '<Model from {0}>'.format(os.path.basename(self.path))
-
- def __reduce__(self):
- return (_kenlm.LanguageModel, (self.path,))
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/setup.py
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/setup.py b/ext/kenlm/setup.py
deleted file mode 100644
index ee061a8..0000000
--- a/ext/kenlm/setup.py
+++ /dev/null
@@ -1,48 +0,0 @@
-from distutils.core import setup
-from distutils.extension import Extension
-import glob
-import platform
-import os
-
-#Does gcc compile with this header and library?
-def compile_test(header, library):
- dummy_path = os.path.join(os.path.dirname(__file__), "dummy")
- command = "bash -c \"g++ -include " + header + " -l" + library + " -x c++ - <<<'int main() {}' -o " + dummy_path + " >/dev/null 2>/dev/null && rm " + dummy_path + " 2>/dev/null\""
- return os.system(command) == 0
-
-
-FILES = glob.glob('util/*.cc') + glob.glob('lm/*.cc') + glob.glob('util/double-conversion/*.cc')
-FILES = [fn for fn in FILES if not (fn.endswith('main.cc') or fn.endswith('test.cc'))]
-
-LIBS = ['stdc++']
-if platform.system() != 'Darwin':
- LIBS.append('rt')
-
-
-ARGS = ['-O3', '-DNDEBUG', '-DKENLM_MAX_ORDER=6']
-
-if compile_test('zlib.h', 'z'):
- ARGS.append('-DHAVE_ZLIB')
- LIBS.append('z')
-
-if compile_test('bzlib.h', 'bz2'):
- ARGS.append('-DHAVE_BZLIB')
- LIBS.append('bz2')
-
-if compile_test('lzma.h', 'lzma'):
- ARGS.append('-DHAVE_XZLIB')
- LIBS.append('lzma')
-
-ext_modules = [
- Extension(name='kenlm',
- sources=FILES + ['python/kenlm.cpp'],
- language='C++',
- include_dirs=['.'],
- libraries=LIBS,
- extra_compile_args=ARGS)
-]
-
-setup(
- name='kenlm',
- ext_modules=ext_modules
-)
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/CMakeLists.txt b/ext/kenlm/util/CMakeLists.txt
deleted file mode 100644
index 8a544aa..0000000
--- a/ext/kenlm/util/CMakeLists.txt
+++ /dev/null
@@ -1,81 +0,0 @@
-cmake_minimum_required(VERSION 2.8.8)
-#
-# The KenLM cmake files make use of add_library(... OBJECTS ...)
-#
-# This syntax allows grouping of source files when compiling
-# (effectively creating "fake" libraries based on source subdirs).
-#
-# This syntax was only added in cmake version 2.8.8
-#
-# see http://www.cmake.org/Wiki/CMake/Tutorials/Object_Library
-
-
-# This CMake file was created by Lane Schwartz <do...@gmail.com>
-
-
-# Explicitly list the source files for this subdirectory
-#
-# If you add any source files to this subdirectory
-# that should be included in the kenlm library,
-# (this excludes any unit test files)
-# you should add them to the following list:
-#
-# Because we do not set PARENT_SCOPE in the following definition,
-# CMake files in the parent directory won't be able to access this variable.
-#
-set(KENLM_UTIL_SOURCE
- bit_packing.cc
- ersatz_progress.cc
- exception.cc
- file.cc
- file_piece.cc
- float_to_string.cc
- integer_to_string.cc
- mmap.cc
- murmur_hash.cc
- parallel_read.cc
- pool.cc
- read_compressed.cc
- scoped.cc
- string_piece.cc
- usage.cc
- )
-
-# This directory has children that need to be processed
-add_subdirectory(double-conversion)
-add_subdirectory(stream)
-
-
-# Group these objects together for later use.
-#
-# Given add_library(foo OBJECT ${my_foo_sources}),
-# refer to these objects as $<TARGET_OBJECTS:foo>
-#
-add_library(kenlm_util OBJECT ${KENLM_UTIL_DOUBLECONVERSION_SOURCE} ${KENLM_UTIL_STREAM_SOURCE} ${KENLM_UTIL_SOURCE})
-
-
-
-# Only compile and run unit tests if tests should be run
-if(BUILD_TESTING)
-
- # Explicitly list the Boost test files to be compiled
- set(KENLM_BOOST_TESTS_LIST
- bit_packing_test
- joint_sort_test
- multi_intersection_test
- probing_hash_table_test
- read_compressed_test
- sorted_uniform_test
- tokenize_piece_test
- )
-
- AddTests(TESTS ${KENLM_BOOST_TESTS_LIST}
- DEPENDS $<TARGET_OBJECTS:kenlm_util>
- LIBRARIES ${Boost_LIBRARIES} pthread)
-
- # file_piece_test requires an extra command line parameter
- KenLMAddTest(TEST file_piece_test
- DEPENDS $<TARGET_OBJECTS:kenlm_util>
- LIBRARIES ${Boost_LIBRARIES} pthread
- TEST_ARGS ${CMAKE_CURRENT_SOURCE_DIR}/file_piece.cc)
-endif()
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/Jamfile
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/Jamfile b/ext/kenlm/util/Jamfile
deleted file mode 100644
index 2eeccf4..0000000
--- a/ext/kenlm/util/Jamfile
+++ /dev/null
@@ -1,41 +0,0 @@
-local compressed_flags = <include>.. <define>HAVE_ZLIB ;
-local compressed_deps = /top//z ;
-if [ test_library "bz2" ] && [ test_header "bzlib.h" ] {
- external-lib bz2 ;
- compressed_flags += <define>HAVE_BZLIB ;
- compressed_deps += bz2 ;
-}
-if [ test_library "lzma" ] && [ test_header "lzma.h" ] {
- external-lib lzma ;
- compressed_flags += <define>HAVE_XZLIB ;
- compressed_deps += lzma ;
-}
-
-#rt is needed for clock_gettime on linux. But it's already included with threading=multi
-lib rt ;
-
-obj read_compressed.o : read_compressed.cc : $(compressed_flags) ;
-alias read_compressed : read_compressed.o $(compressed_deps) ;
-obj read_compressed_test.o : read_compressed_test.cc /top//boost_unit_test_framework : $(compressed_flags) ;
-obj file_piece_test.o : file_piece_test.cc /top//boost_unit_test_framework : $(compressed_flags) ;
-
-fakelib parallel_read : parallel_read.cc : <threading>multi:<source>/top//boost_thread <threading>multi:<define>WITH_THREADS : : <include>.. ;
-
-fakelib kenutil : [ glob *.cc : parallel_read.cc read_compressed.cc *_main.cc *_test.cc ] read_compressed parallel_read double-conversion//double-conversion : <include>.. <os>LINUX,<threading>single:<source>rt : : <include>.. ;
-
-exe cat_compressed : cat_compressed_main.cc kenutil ;
-
-#Does not install this
-exe probing_hash_table_benchmark : probing_hash_table_benchmark_main.cc kenutil ;
-
-alias programs : cat_compressed ;
-
-import testing ;
-
-run file_piece_test.o kenutil /top//boost_unit_test_framework : : file_piece.cc ;
-for local t in [ glob *_test.cc : file_piece_test.cc read_compressed_test.cc ] {
- local name = [ MATCH "(.*)\.cc" : $(t) ] ;
- unit-test $(name) : $(t) kenutil /top//boost_unit_test_framework /top//boost_system ;
-}
-
-build-project stream ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/bit_packing.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/bit_packing.cc b/ext/kenlm/util/bit_packing.cc
deleted file mode 100644
index cffd9cf..0000000
--- a/ext/kenlm/util/bit_packing.cc
+++ /dev/null
@@ -1,40 +0,0 @@
-#include "util/bit_packing.hh"
-#include "util/exception.hh"
-
-#include <cstring>
-
-namespace util {
-
-namespace {
-template <bool> struct StaticCheck {};
-template <> struct StaticCheck<true> { typedef bool StaticAssertionPassed; };
-
-// If your float isn't 4 bytes, we're hosed.
-typedef StaticCheck<sizeof(float) == 4>::StaticAssertionPassed FloatSize;
-
-} // namespace
-
-uint8_t RequiredBits(uint64_t max_value) {
- if (!max_value) return 0;
- uint8_t ret = 1;
- while (max_value >>= 1) ++ret;
- return ret;
-}
-
-void BitPackingSanity() {
- const FloatEnc neg1 = { -1.0 }, pos1 = { 1.0 };
- if ((neg1.i ^ pos1.i) != 0x80000000) UTIL_THROW(Exception, "Sign bit is not 0x80000000");
- char mem[57+8];
- memset(mem, 0, sizeof(mem));
- const uint64_t test57 = 0x123456789abcdefULL;
- for (uint64_t b = 0; b < 57 * 8; b += 57) {
- WriteInt57(mem, b, 57, test57);
- }
- for (uint64_t b = 0; b < 57 * 8; b += 57) {
- if (test57 != ReadInt57(mem, b, 57, (1ULL << 57) - 1))
- UTIL_THROW(Exception, "The bit packing routines are failing for your architecture. Please send a bug report with your architecture, operating system, and compiler.");
- }
- // TODO: more checks.
-}
-
-} // namespace util
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/bit_packing.hh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/bit_packing.hh b/ext/kenlm/util/bit_packing.hh
deleted file mode 100644
index b24fd9c..0000000
--- a/ext/kenlm/util/bit_packing.hh
+++ /dev/null
@@ -1,185 +0,0 @@
-#ifndef UTIL_BIT_PACKING_H
-#define UTIL_BIT_PACKING_H
-
-/* Bit-level packing routines
- *
- * WARNING WARNING WARNING:
- * The write functions assume that memory is zero initially. This makes them
- * faster and is the appropriate case for mmapped language model construction.
- * These routines assume that unaligned access to uint64_t is fast. This is
- * the case on x86_64. I'm not sure how fast unaligned 64-bit access is on
- * x86 but my target audience is large language models for which 64-bit is
- * necessary.
- *
- * Call the BitPackingSanity function to sanity check. Calling once suffices,
- * but it may be called multiple times when that's inconvenient.
- *
- * ARM and MinGW ports contributed by Hideo Okuma and Tomoyuki Yoshimura at
- * NICT.
- */
-
-#include <cassert>
-#ifdef __APPLE__
-#include <architecture/byte_order.h>
-#elif __linux__
-#include <endian.h>
-#elif !defined(_WIN32) && !defined(_WIN64)
-#include <arpa/nameser_compat.h>
-#endif
-
-#include <stdint.h>
-#include <cstring>
-
-namespace util {
-
-// Fun fact: __BYTE_ORDER is wrong on Solaris Sparc, but the version without __ is correct.
-#if BYTE_ORDER == LITTLE_ENDIAN
-inline uint8_t BitPackShift(uint8_t bit, uint8_t /*length*/) {
- return bit;
-}
-#elif BYTE_ORDER == BIG_ENDIAN
-inline uint8_t BitPackShift(uint8_t bit, uint8_t length) {
- return 64 - length - bit;
-}
-#else
-#error "Bit packing code isn't written for your byte order."
-#endif
-
-inline uint64_t ReadOff(const void *base, uint64_t bit_off) {
-#if defined(__arm) || defined(__arm__)
- const uint8_t *base_off = reinterpret_cast<const uint8_t*>(base) + (bit_off >> 3);
- uint64_t value64;
- memcpy(&value64, base_off, sizeof(value64));
- return value64;
-#else
- return *reinterpret_cast<const uint64_t*>(reinterpret_cast<const uint8_t*>(base) + (bit_off >> 3));
-#endif
-}
-
-/* Pack integers up to 57 bits using their least significant digits.
- * The length is specified using mask:
- * Assumes mask == (1 << length) - 1 where length <= 57.
- */
-inline uint64_t ReadInt57(const void *base, uint64_t bit_off, uint8_t length, uint64_t mask) {
- return (ReadOff(base, bit_off) >> BitPackShift(bit_off & 7, length)) & mask;
-}
-/* Assumes value < (1 << length) and length <= 57.
- * Assumes the memory is zero initially.
- */
-inline void WriteInt57(void *base, uint64_t bit_off, uint8_t length, uint64_t value) {
-#if defined(__arm) || defined(__arm__)
- uint8_t *base_off = reinterpret_cast<uint8_t*>(base) + (bit_off >> 3);
- uint64_t value64;
- memcpy(&value64, base_off, sizeof(value64));
- value64 |= (value << BitPackShift(bit_off & 7, length));
- memcpy(base_off, &value64, sizeof(value64));
-#else
- *reinterpret_cast<uint64_t*>(reinterpret_cast<uint8_t*>(base) + (bit_off >> 3)) |=
- (value << BitPackShift(bit_off & 7, length));
-#endif
-}
-
-/* Same caveats as above, but for a 25 bit limit. */
-inline uint32_t ReadInt25(const void *base, uint64_t bit_off, uint8_t length, uint32_t mask) {
-#if defined(__arm) || defined(__arm__)
- const uint8_t *base_off = reinterpret_cast<const uint8_t*>(base) + (bit_off >> 3);
- uint32_t value32;
- memcpy(&value32, base_off, sizeof(value32));
- return (value32 >> BitPackShift(bit_off & 7, length)) & mask;
-#else
- return (*reinterpret_cast<const uint32_t*>(reinterpret_cast<const uint8_t*>(base) + (bit_off >> 3)) >> BitPackShift(bit_off & 7, length)) & mask;
-#endif
-}
-
-inline void WriteInt25(void *base, uint64_t bit_off, uint8_t length, uint32_t value) {
-#if defined(__arm) || defined(__arm__)
- uint8_t *base_off = reinterpret_cast<uint8_t*>(base) + (bit_off >> 3);
- uint32_t value32;
- memcpy(&value32, base_off, sizeof(value32));
- value32 |= (value << BitPackShift(bit_off & 7, length));
- memcpy(base_off, &value32, sizeof(value32));
-#else
- *reinterpret_cast<uint32_t*>(reinterpret_cast<uint8_t*>(base) + (bit_off >> 3)) |=
- (value << BitPackShift(bit_off & 7, length));
-#endif
-}
-
-typedef union { float f; uint32_t i; } FloatEnc;
-
-inline float ReadFloat32(const void *base, uint64_t bit_off) {
- FloatEnc encoded;
- encoded.i = ReadOff(base, bit_off) >> BitPackShift(bit_off & 7, 32);
- return encoded.f;
-}
-inline void WriteFloat32(void *base, uint64_t bit_off, float value) {
- FloatEnc encoded;
- encoded.f = value;
- WriteInt57(base, bit_off, 32, encoded.i);
-}
-
-const uint32_t kSignBit = 0x80000000;
-
-inline void SetSign(float &to) {
- FloatEnc enc;
- enc.f = to;
- enc.i |= kSignBit;
- to = enc.f;
-}
-
-inline void UnsetSign(float &to) {
- FloatEnc enc;
- enc.f = to;
- enc.i &= ~kSignBit;
- to = enc.f;
-}
-
-inline float ReadNonPositiveFloat31(const void *base, uint64_t bit_off) {
- FloatEnc encoded;
- encoded.i = ReadOff(base, bit_off) >> BitPackShift(bit_off & 7, 31);
- // Sign bit set means negative.
- encoded.i |= kSignBit;
- return encoded.f;
-}
-inline void WriteNonPositiveFloat31(void *base, uint64_t bit_off, float value) {
- FloatEnc encoded;
- encoded.f = value;
- encoded.i &= ~kSignBit;
- WriteInt57(base, bit_off, 31, encoded.i);
-}
-
-void BitPackingSanity();
-
-// Return bits required to store integers upto max_value. Not the most
-// efficient implementation, but this is only called a few times to size tries.
-uint8_t RequiredBits(uint64_t max_value);
-
-struct BitsMask {
- static BitsMask ByMax(uint64_t max_value) {
- BitsMask ret;
- ret.FromMax(max_value);
- return ret;
- }
- static BitsMask ByBits(uint8_t bits) {
- BitsMask ret;
- ret.bits = bits;
- ret.mask = (1ULL << bits) - 1;
- return ret;
- }
- void FromMax(uint64_t max_value) {
- bits = RequiredBits(max_value);
- mask = (1ULL << bits) - 1;
- }
- uint8_t bits;
- uint64_t mask;
-};
-
-struct BitAddress {
- BitAddress(void *in_base, uint64_t in_offset) : base(in_base), offset(in_offset) {}
-
- void *base;
- uint64_t offset;
-};
-
-} // namespace util
-
-#endif // UTIL_BIT_PACKING_H
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/bit_packing_test.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/bit_packing_test.cc b/ext/kenlm/util/bit_packing_test.cc
deleted file mode 100644
index c4494b6..0000000
--- a/ext/kenlm/util/bit_packing_test.cc
+++ /dev/null
@@ -1,59 +0,0 @@
-#include "util/bit_packing.hh"
-
-#define BOOST_TEST_MODULE BitPackingTest
-#include <boost/test/unit_test.hpp>
-
-#include <cstring>
-
-namespace util {
-namespace {
-
-const uint64_t test57 = 0x123456789abcdefULL;
-const uint32_t test25 = 0x1234567;
-
-BOOST_AUTO_TEST_CASE(ZeroBit57) {
- char mem[16];
- memset(mem, 0, sizeof(mem));
- WriteInt57(mem, 0, 57, test57);
- BOOST_CHECK_EQUAL(test57, ReadInt57(mem, 0, 57, (1ULL << 57) - 1));
-}
-
-BOOST_AUTO_TEST_CASE(EachBit57) {
- char mem[16];
- for (uint8_t b = 0; b < 8; ++b) {
- memset(mem, 0, sizeof(mem));
- WriteInt57(mem, b, 57, test57);
- BOOST_CHECK_EQUAL(test57, ReadInt57(mem, b, 57, (1ULL << 57) - 1));
- }
-}
-
-BOOST_AUTO_TEST_CASE(Consecutive57) {
- char mem[57+8];
- memset(mem, 0, sizeof(mem));
- for (uint64_t b = 0; b < 57 * 8; b += 57) {
- WriteInt57(mem, b, 57, test57);
- BOOST_CHECK_EQUAL(test57, ReadInt57(mem, b, 57, (1ULL << 57) - 1));
- }
- for (uint64_t b = 0; b < 57 * 8; b += 57) {
- BOOST_CHECK_EQUAL(test57, ReadInt57(mem, b, 57, (1ULL << 57) - 1));
- }
-}
-
-BOOST_AUTO_TEST_CASE(Consecutive25) {
- char mem[25+8];
- memset(mem, 0, sizeof(mem));
- for (uint64_t b = 0; b < 25 * 8; b += 25) {
- WriteInt25(mem, b, 25, test25);
- BOOST_CHECK_EQUAL(test25, ReadInt25(mem, b, 25, (1ULL << 25) - 1));
- }
- for (uint64_t b = 0; b < 25 * 8; b += 25) {
- BOOST_CHECK_EQUAL(test25, ReadInt25(mem, b, 25, (1ULL << 25) - 1));
- }
-}
-
-BOOST_AUTO_TEST_CASE(Sanity) {
- BitPackingSanity();
-}
-
-} // namespace
-} // namespace util
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/cat_compressed_main.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/cat_compressed_main.cc b/ext/kenlm/util/cat_compressed_main.cc
deleted file mode 100644
index 0c7cda9..0000000
--- a/ext/kenlm/util/cat_compressed_main.cc
+++ /dev/null
@@ -1,47 +0,0 @@
-// Like cat but interprets compressed files.
-#include "util/file.hh"
-#include "util/read_compressed.hh"
-
-#include <cstring>
-#include <iostream>
-
-namespace {
-const std::size_t kBufSize = 16384;
-void Copy(util::ReadCompressed &from, int to) {
- util::scoped_malloc buffer(util::MallocOrThrow(kBufSize));
- while (std::size_t amount = from.Read(buffer.get(), kBufSize)) {
- util::WriteOrThrow(to, buffer.get(), amount);
- }
-}
-} // namespace
-
-int main(int argc, char *argv[]) {
- // Lane Schwartz likes -h and --help
- for (int i = 1; i < argc; ++i) {
- char *arg = argv[i];
- if (!strcmp(arg, "--")) break;
- if (!strcmp(arg, "-h") || !strcmp(arg, "--help")) {
- std::cerr <<
- "A cat implementation that interprets compressed files.\n"
- "Usage: " << argv[0] << " [file1] [file2] ...\n"
- "If no file is provided, then stdin is read.\n";
- return 1;
- }
- }
-
- try {
- if (argc == 1) {
- util::ReadCompressed in(0);
- Copy(in, 1);
- } else {
- for (int i = 1; i < argc; ++i) {
- util::ReadCompressed in(util::OpenReadOrThrow(argv[i]));
- Copy(in, 1);
- }
- }
- } catch (const std::exception &e) {
- std::cerr << e.what() << std::endl;
- return 2;
- }
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/double-conversion/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/double-conversion/CMakeLists.txt b/ext/kenlm/util/double-conversion/CMakeLists.txt
deleted file mode 100644
index e2cf02a..0000000
--- a/ext/kenlm/util/double-conversion/CMakeLists.txt
+++ /dev/null
@@ -1,39 +0,0 @@
-cmake_minimum_required(VERSION 2.8.8)
-#
-# The KenLM cmake files make use of add_library(... OBJECTS ...)
-#
-# This syntax allows grouping of source files when compiling
-# (effectively creating "fake" libraries based on source subdirs).
-#
-# This syntax was only added in cmake version 2.8.8
-#
-# see http://www.cmake.org/Wiki/CMake/Tutorials/Object_Library
-
-
-# This CMake file was created by Lane Schwartz <do...@gmail.com>
-
-# Explicitly list the source files for this subdirectory
-#
-# If you add any source files to this subdirectory
-# that should be included in the kenlm library,
-# (this excludes any unit test files)
-# you should add them to the following list:
-#
-# In order to allow CMake files in the parent directory
-# to see this variable definition, we set PARENT_SCOPE.
-#
-# In order to set correct paths to these files
-# when this variable is referenced by CMake files in the parent directory,
-# we prefix all files with ${CMAKE_CURRENT_SOURCE_DIR}.
-#
-set(KENLM_UTIL_DOUBLECONVERSION_SOURCE
- ${CMAKE_CURRENT_SOURCE_DIR}/bignum-dtoa.cc
- ${CMAKE_CURRENT_SOURCE_DIR}/bignum.cc
- ${CMAKE_CURRENT_SOURCE_DIR}/cached-powers.cc
- ${CMAKE_CURRENT_SOURCE_DIR}/diy-fp.cc
- ${CMAKE_CURRENT_SOURCE_DIR}/double-conversion.cc
- ${CMAKE_CURRENT_SOURCE_DIR}/fast-dtoa.cc
- ${CMAKE_CURRENT_SOURCE_DIR}/fixed-dtoa.cc
- ${CMAKE_CURRENT_SOURCE_DIR}/strtod.cc
- PARENT_SCOPE)
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/double-conversion/Jamfile
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/double-conversion/Jamfile b/ext/kenlm/util/double-conversion/Jamfile
deleted file mode 100644
index 7a92afa..0000000
--- a/ext/kenlm/util/double-conversion/Jamfile
+++ /dev/null
@@ -1 +0,0 @@
-fakelib double-conversion : [ glob *.cc ] : : : <include>. ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/double-conversion/LICENSE
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/double-conversion/LICENSE b/ext/kenlm/util/double-conversion/LICENSE
deleted file mode 100644
index 933718a..0000000
--- a/ext/kenlm/util/double-conversion/LICENSE
+++ /dev/null
@@ -1,26 +0,0 @@
-Copyright 2006-2011, the V8 project authors. All rights reserved.
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following
- disclaimer in the documentation and/or other materials provided
- with the distribution.
- * Neither the name of Google Inc. nor the names of its
- contributors may be used to endorse or promote products derived
- from this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/double-conversion/bignum-dtoa.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/double-conversion/bignum-dtoa.cc b/ext/kenlm/util/double-conversion/bignum-dtoa.cc
deleted file mode 100644
index 3d217bf..0000000
--- a/ext/kenlm/util/double-conversion/bignum-dtoa.cc
+++ /dev/null
@@ -1,640 +0,0 @@
-// Copyright 2010 the V8 project authors. All rights reserved.
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following
-// disclaimer in the documentation and/or other materials provided
-// with the distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived
-// from this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#include <cmath>
-
-#include "bignum-dtoa.h"
-
-#include "bignum.h"
-#include "ieee.h"
-
-namespace double_conversion {
-
-static int NormalizedExponent(uint64_t significand, int exponent) {
- ASSERT(significand != 0);
- while ((significand & Double::kHiddenBit) == 0) {
- significand = significand << 1;
- exponent = exponent - 1;
- }
- return exponent;
-}
-
-
-// Forward declarations:
-// Returns an estimation of k such that 10^(k-1) <= v < 10^k.
-static int EstimatePower(int exponent);
-// Computes v / 10^estimated_power exactly, as a ratio of two bignums, numerator
-// and denominator.
-static void InitialScaledStartValues(uint64_t significand,
- int exponent,
- bool lower_boundary_is_closer,
- int estimated_power,
- bool need_boundary_deltas,
- Bignum* numerator,
- Bignum* denominator,
- Bignum* delta_minus,
- Bignum* delta_plus);
-// Multiplies numerator/denominator so that its values lies in the range 1-10.
-// Returns decimal_point s.t.
-// v = numerator'/denominator' * 10^(decimal_point-1)
-// where numerator' and denominator' are the values of numerator and
-// denominator after the call to this function.
-static void FixupMultiply10(int estimated_power, bool is_even,
- int* decimal_point,
- Bignum* numerator, Bignum* denominator,
- Bignum* delta_minus, Bignum* delta_plus);
-// Generates digits from the left to the right and stops when the generated
-// digits yield the shortest decimal representation of v.
-static void GenerateShortestDigits(Bignum* numerator, Bignum* denominator,
- Bignum* delta_minus, Bignum* delta_plus,
- bool is_even,
- Vector<char> buffer, int* length);
-// Generates 'requested_digits' after the decimal point.
-static void BignumToFixed(int requested_digits, int* decimal_point,
- Bignum* numerator, Bignum* denominator,
- Vector<char>(buffer), int* length);
-// Generates 'count' digits of numerator/denominator.
-// Once 'count' digits have been produced rounds the result depending on the
-// remainder (remainders of exactly .5 round upwards). Might update the
-// decimal_point when rounding up (for example for 0.9999).
-static void GenerateCountedDigits(int count, int* decimal_point,
- Bignum* numerator, Bignum* denominator,
- Vector<char>(buffer), int* length);
-
-
-void BignumDtoa(double v, BignumDtoaMode mode, int requested_digits,
- Vector<char> buffer, int* length, int* decimal_point) {
- ASSERT(v > 0);
- ASSERT(!Double(v).IsSpecial());
- uint64_t significand;
- int exponent;
- bool lower_boundary_is_closer;
- if (mode == BIGNUM_DTOA_SHORTEST_SINGLE) {
- float f = static_cast<float>(v);
- ASSERT(f == v);
- significand = Single(f).Significand();
- exponent = Single(f).Exponent();
- lower_boundary_is_closer = Single(f).LowerBoundaryIsCloser();
- } else {
- significand = Double(v).Significand();
- exponent = Double(v).Exponent();
- lower_boundary_is_closer = Double(v).LowerBoundaryIsCloser();
- }
- bool need_boundary_deltas =
- (mode == BIGNUM_DTOA_SHORTEST || mode == BIGNUM_DTOA_SHORTEST_SINGLE);
-
- bool is_even = (significand & 1) == 0;
- int normalized_exponent = NormalizedExponent(significand, exponent);
- // estimated_power might be too low by 1.
- int estimated_power = EstimatePower(normalized_exponent);
-
- // Shortcut for Fixed.
- // The requested digits correspond to the digits after the point. If the
- // number is much too small, then there is no need in trying to get any
- // digits.
- if (mode == BIGNUM_DTOA_FIXED && -estimated_power - 1 > requested_digits) {
- buffer[0] = '\0';
- *length = 0;
- // Set decimal-point to -requested_digits. This is what Gay does.
- // Note that it should not have any effect anyways since the string is
- // empty.
- *decimal_point = -requested_digits;
- return;
- }
-
- Bignum numerator;
- Bignum denominator;
- Bignum delta_minus;
- Bignum delta_plus;
- // Make sure the bignum can grow large enough. The smallest double equals
- // 4e-324. In this case the denominator needs fewer than 324*4 binary digits.
- // The maximum double is 1.7976931348623157e308 which needs fewer than
- // 308*4 binary digits.
- ASSERT(Bignum::kMaxSignificantBits >= 324*4);
- InitialScaledStartValues(significand, exponent, lower_boundary_is_closer,
- estimated_power, need_boundary_deltas,
- &numerator, &denominator,
- &delta_minus, &delta_plus);
- // We now have v = (numerator / denominator) * 10^estimated_power.
- FixupMultiply10(estimated_power, is_even, decimal_point,
- &numerator, &denominator,
- &delta_minus, &delta_plus);
- // We now have v = (numerator / denominator) * 10^(decimal_point-1), and
- // 1 <= (numerator + delta_plus) / denominator < 10
- switch (mode) {
- case BIGNUM_DTOA_SHORTEST:
- case BIGNUM_DTOA_SHORTEST_SINGLE:
- GenerateShortestDigits(&numerator, &denominator,
- &delta_minus, &delta_plus,
- is_even, buffer, length);
- break;
- case BIGNUM_DTOA_FIXED:
- BignumToFixed(requested_digits, decimal_point,
- &numerator, &denominator,
- buffer, length);
- break;
- case BIGNUM_DTOA_PRECISION:
- GenerateCountedDigits(requested_digits, decimal_point,
- &numerator, &denominator,
- buffer, length);
- break;
- default:
- UNREACHABLE();
- }
- buffer[*length] = '\0';
-}
-
-
-// The procedure starts generating digits from the left to the right and stops
-// when the generated digits yield the shortest decimal representation of v. A
-// decimal representation of v is a number lying closer to v than to any other
-// double, so it converts to v when read.
-//
-// This is true if d, the decimal representation, is between m- and m+, the
-// upper and lower boundaries. d must be strictly between them if !is_even.
-// m- := (numerator - delta_minus) / denominator
-// m+ := (numerator + delta_plus) / denominator
-//
-// Precondition: 0 <= (numerator+delta_plus) / denominator < 10.
-// If 1 <= (numerator+delta_plus) / denominator < 10 then no leading 0 digit
-// will be produced. This should be the standard precondition.
-static void GenerateShortestDigits(Bignum* numerator, Bignum* denominator,
- Bignum* delta_minus, Bignum* delta_plus,
- bool is_even,
- Vector<char> buffer, int* length) {
- // Small optimization: if delta_minus and delta_plus are the same just reuse
- // one of the two bignums.
- if (Bignum::Equal(*delta_minus, *delta_plus)) {
- delta_plus = delta_minus;
- }
- *length = 0;
- while (true) {
- uint16_t digit;
- digit = numerator->DivideModuloIntBignum(*denominator);
- ASSERT(digit <= 9); // digit is a uint16_t and therefore always positive.
- // digit = numerator / denominator (integer division).
- // numerator = numerator % denominator.
- buffer[(*length)++] = digit + '0';
-
- // Can we stop already?
- // If the remainder of the division is less than the distance to the lower
- // boundary we can stop. In this case we simply round down (discarding the
- // remainder).
- // Similarly we test if we can round up (using the upper boundary).
- bool in_delta_room_minus;
- bool in_delta_room_plus;
- if (is_even) {
- in_delta_room_minus = Bignum::LessEqual(*numerator, *delta_minus);
- } else {
- in_delta_room_minus = Bignum::Less(*numerator, *delta_minus);
- }
- if (is_even) {
- in_delta_room_plus =
- Bignum::PlusCompare(*numerator, *delta_plus, *denominator) >= 0;
- } else {
- in_delta_room_plus =
- Bignum::PlusCompare(*numerator, *delta_plus, *denominator) > 0;
- }
- if (!in_delta_room_minus && !in_delta_room_plus) {
- // Prepare for next iteration.
- numerator->Times10();
- delta_minus->Times10();
- // We optimized delta_plus to be equal to delta_minus (if they share the
- // same value). So don't multiply delta_plus if they point to the same
- // object.
- if (delta_minus != delta_plus) {
- delta_plus->Times10();
- }
- } else if (in_delta_room_minus && in_delta_room_plus) {
- // Let's see if 2*numerator < denominator.
- // If yes, then the next digit would be < 5 and we can round down.
- int compare = Bignum::PlusCompare(*numerator, *numerator, *denominator);
- if (compare < 0) {
- // Remaining digits are less than .5. -> Round down (== do nothing).
- } else if (compare > 0) {
- // Remaining digits are more than .5 of denominator. -> Round up.
- // Note that the last digit could not be a '9' as otherwise the whole
- // loop would have stopped earlier.
- // We still have an assert here in case the preconditions were not
- // satisfied.
- ASSERT(buffer[(*length) - 1] != '9');
- buffer[(*length) - 1]++;
- } else {
- // Halfway case.
- // TODO(floitsch): need a way to solve half-way cases.
- // For now let's round towards even (since this is what Gay seems to
- // do).
-
- if ((buffer[(*length) - 1] - '0') % 2 == 0) {
- // Round down => Do nothing.
- } else {
- ASSERT(buffer[(*length) - 1] != '9');
- buffer[(*length) - 1]++;
- }
- }
- return;
- } else if (in_delta_room_minus) {
- // Round down (== do nothing).
- return;
- } else { // in_delta_room_plus
- // Round up.
- // Note again that the last digit could not be '9' since this would have
- // stopped the loop earlier.
- // We still have an ASSERT here, in case the preconditions were not
- // satisfied.
- ASSERT(buffer[(*length) -1] != '9');
- buffer[(*length) - 1]++;
- return;
- }
- }
-}
-
-
-// Let v = numerator / denominator < 10.
-// Then we generate 'count' digits of d = x.xxxxx... (without the decimal point)
-// from left to right. Once 'count' digits have been produced we decide wether
-// to round up or down. Remainders of exactly .5 round upwards. Numbers such
-// as 9.999999 propagate a carry all the way, and change the
-// exponent (decimal_point), when rounding upwards.
-static void GenerateCountedDigits(int count, int* decimal_point,
- Bignum* numerator, Bignum* denominator,
- Vector<char>(buffer), int* length) {
- ASSERT(count >= 0);
- for (int i = 0; i < count - 1; ++i) {
- uint16_t digit;
- digit = numerator->DivideModuloIntBignum(*denominator);
- ASSERT(digit <= 9); // digit is a uint16_t and therefore always positive.
- // digit = numerator / denominator (integer division).
- // numerator = numerator % denominator.
- buffer[i] = digit + '0';
- // Prepare for next iteration.
- numerator->Times10();
- }
- // Generate the last digit.
- uint16_t digit;
- digit = numerator->DivideModuloIntBignum(*denominator);
- if (Bignum::PlusCompare(*numerator, *numerator, *denominator) >= 0) {
- digit++;
- }
- buffer[count - 1] = digit + '0';
- // Correct bad digits (in case we had a sequence of '9's). Propagate the
- // carry until we hat a non-'9' or til we reach the first digit.
- for (int i = count - 1; i > 0; --i) {
- if (buffer[i] != '0' + 10) break;
- buffer[i] = '0';
- buffer[i - 1]++;
- }
- if (buffer[0] == '0' + 10) {
- // Propagate a carry past the top place.
- buffer[0] = '1';
- (*decimal_point)++;
- }
- *length = count;
-}
-
-
-// Generates 'requested_digits' after the decimal point. It might omit
-// trailing '0's. If the input number is too small then no digits at all are
-// generated (ex.: 2 fixed digits for 0.00001).
-//
-// Input verifies: 1 <= (numerator + delta) / denominator < 10.
-static void BignumToFixed(int requested_digits, int* decimal_point,
- Bignum* numerator, Bignum* denominator,
- Vector<char>(buffer), int* length) {
- // Note that we have to look at more than just the requested_digits, since
- // a number could be rounded up. Example: v=0.5 with requested_digits=0.
- // Even though the power of v equals 0 we can't just stop here.
- if (-(*decimal_point) > requested_digits) {
- // The number is definitively too small.
- // Ex: 0.001 with requested_digits == 1.
- // Set decimal-point to -requested_digits. This is what Gay does.
- // Note that it should not have any effect anyways since the string is
- // empty.
- *decimal_point = -requested_digits;
- *length = 0;
- return;
- } else if (-(*decimal_point) == requested_digits) {
- // We only need to verify if the number rounds down or up.
- // Ex: 0.04 and 0.06 with requested_digits == 1.
- ASSERT(*decimal_point == -requested_digits);
- // Initially the fraction lies in range (1, 10]. Multiply the denominator
- // by 10 so that we can compare more easily.
- denominator->Times10();
- if (Bignum::PlusCompare(*numerator, *numerator, *denominator) >= 0) {
- // If the fraction is >= 0.5 then we have to include the rounded
- // digit.
- buffer[0] = '1';
- *length = 1;
- (*decimal_point)++;
- } else {
- // Note that we caught most of similar cases earlier.
- *length = 0;
- }
- return;
- } else {
- // The requested digits correspond to the digits after the point.
- // The variable 'needed_digits' includes the digits before the point.
- int needed_digits = (*decimal_point) + requested_digits;
- GenerateCountedDigits(needed_digits, decimal_point,
- numerator, denominator,
- buffer, length);
- }
-}
-
-
-// Returns an estimation of k such that 10^(k-1) <= v < 10^k where
-// v = f * 2^exponent and 2^52 <= f < 2^53.
-// v is hence a normalized double with the given exponent. The output is an
-// approximation for the exponent of the decimal approimation .digits * 10^k.
-//
-// The result might undershoot by 1 in which case 10^k <= v < 10^k+1.
-// Note: this property holds for v's upper boundary m+ too.
-// 10^k <= m+ < 10^k+1.
-// (see explanation below).
-//
-// Examples:
-// EstimatePower(0) => 16
-// EstimatePower(-52) => 0
-//
-// Note: e >= 0 => EstimatedPower(e) > 0. No similar claim can be made for e<0.
-static int EstimatePower(int exponent) {
- // This function estimates log10 of v where v = f*2^e (with e == exponent).
- // Note that 10^floor(log10(v)) <= v, but v <= 10^ceil(log10(v)).
- // Note that f is bounded by its container size. Let p = 53 (the double's
- // significand size). Then 2^(p-1) <= f < 2^p.
- //
- // Given that log10(v) == log2(v)/log2(10) and e+(len(f)-1) is quite close
- // to log2(v) the function is simplified to (e+(len(f)-1)/log2(10)).
- // The computed number undershoots by less than 0.631 (when we compute log3
- // and not log10).
- //
- // Optimization: since we only need an approximated result this computation
- // can be performed on 64 bit integers. On x86/x64 architecture the speedup is
- // not really measurable, though.
- //
- // Since we want to avoid overshooting we decrement by 1e10 so that
- // floating-point imprecisions don't affect us.
- //
- // Explanation for v's boundary m+: the computation takes advantage of
- // the fact that 2^(p-1) <= f < 2^p. Boundaries still satisfy this requirement
- // (even for denormals where the delta can be much more important).
-
- const double k1Log10 = 0.30102999566398114; // 1/lg(10)
-
- // For doubles len(f) == 53 (don't forget the hidden bit).
- const int kSignificandSize = Double::kSignificandSize;
- double estimate = ceil((exponent + kSignificandSize - 1) * k1Log10 - 1e-10);
- return static_cast<int>(estimate);
-}
-
-
-// See comments for InitialScaledStartValues.
-static void InitialScaledStartValuesPositiveExponent(
- uint64_t significand, int exponent,
- int estimated_power, bool need_boundary_deltas,
- Bignum* numerator, Bignum* denominator,
- Bignum* delta_minus, Bignum* delta_plus) {
- // A positive exponent implies a positive power.
- ASSERT(estimated_power >= 0);
- // Since the estimated_power is positive we simply multiply the denominator
- // by 10^estimated_power.
-
- // numerator = v.
- numerator->AssignUInt64(significand);
- numerator->ShiftLeft(exponent);
- // denominator = 10^estimated_power.
- denominator->AssignPowerUInt16(10, estimated_power);
-
- if (need_boundary_deltas) {
- // Introduce a common denominator so that the deltas to the boundaries are
- // integers.
- denominator->ShiftLeft(1);
- numerator->ShiftLeft(1);
- // Let v = f * 2^e, then m+ - v = 1/2 * 2^e; With the common
- // denominator (of 2) delta_plus equals 2^e.
- delta_plus->AssignUInt16(1);
- delta_plus->ShiftLeft(exponent);
- // Same for delta_minus. The adjustments if f == 2^p-1 are done later.
- delta_minus->AssignUInt16(1);
- delta_minus->ShiftLeft(exponent);
- }
-}
-
-
-// See comments for InitialScaledStartValues
-static void InitialScaledStartValuesNegativeExponentPositivePower(
- uint64_t significand, int exponent,
- int estimated_power, bool need_boundary_deltas,
- Bignum* numerator, Bignum* denominator,
- Bignum* delta_minus, Bignum* delta_plus) {
- // v = f * 2^e with e < 0, and with estimated_power >= 0.
- // This means that e is close to 0 (have a look at how estimated_power is
- // computed).
-
- // numerator = significand
- // since v = significand * 2^exponent this is equivalent to
- // numerator = v * / 2^-exponent
- numerator->AssignUInt64(significand);
- // denominator = 10^estimated_power * 2^-exponent (with exponent < 0)
- denominator->AssignPowerUInt16(10, estimated_power);
- denominator->ShiftLeft(-exponent);
-
- if (need_boundary_deltas) {
- // Introduce a common denominator so that the deltas to the boundaries are
- // integers.
- denominator->ShiftLeft(1);
- numerator->ShiftLeft(1);
- // Let v = f * 2^e, then m+ - v = 1/2 * 2^e; With the common
- // denominator (of 2) delta_plus equals 2^e.
- // Given that the denominator already includes v's exponent the distance
- // to the boundaries is simply 1.
- delta_plus->AssignUInt16(1);
- // Same for delta_minus. The adjustments if f == 2^p-1 are done later.
- delta_minus->AssignUInt16(1);
- }
-}
-
-
-// See comments for InitialScaledStartValues
-static void InitialScaledStartValuesNegativeExponentNegativePower(
- uint64_t significand, int exponent,
- int estimated_power, bool need_boundary_deltas,
- Bignum* numerator, Bignum* denominator,
- Bignum* delta_minus, Bignum* delta_plus) {
- // Instead of multiplying the denominator with 10^estimated_power we
- // multiply all values (numerator and deltas) by 10^-estimated_power.
-
- // Use numerator as temporary container for power_ten.
- Bignum* power_ten = numerator;
- power_ten->AssignPowerUInt16(10, -estimated_power);
-
- if (need_boundary_deltas) {
- // Since power_ten == numerator we must make a copy of 10^estimated_power
- // before we complete the computation of the numerator.
- // delta_plus = delta_minus = 10^estimated_power
- delta_plus->AssignBignum(*power_ten);
- delta_minus->AssignBignum(*power_ten);
- }
-
- // numerator = significand * 2 * 10^-estimated_power
- // since v = significand * 2^exponent this is equivalent to
- // numerator = v * 10^-estimated_power * 2 * 2^-exponent.
- // Remember: numerator has been abused as power_ten. So no need to assign it
- // to itself.
- ASSERT(numerator == power_ten);
- numerator->MultiplyByUInt64(significand);
-
- // denominator = 2 * 2^-exponent with exponent < 0.
- denominator->AssignUInt16(1);
- denominator->ShiftLeft(-exponent);
-
- if (need_boundary_deltas) {
- // Introduce a common denominator so that the deltas to the boundaries are
- // integers.
- numerator->ShiftLeft(1);
- denominator->ShiftLeft(1);
- // With this shift the boundaries have their correct value, since
- // delta_plus = 10^-estimated_power, and
- // delta_minus = 10^-estimated_power.
- // These assignments have been done earlier.
- // The adjustments if f == 2^p-1 (lower boundary is closer) are done later.
- }
-}
-
-
-// Let v = significand * 2^exponent.
-// Computes v / 10^estimated_power exactly, as a ratio of two bignums, numerator
-// and denominator. The functions GenerateShortestDigits and
-// GenerateCountedDigits will then convert this ratio to its decimal
-// representation d, with the required accuracy.
-// Then d * 10^estimated_power is the representation of v.
-// (Note: the fraction and the estimated_power might get adjusted before
-// generating the decimal representation.)
-//
-// The initial start values consist of:
-// - a scaled numerator: s.t. numerator/denominator == v / 10^estimated_power.
-// - a scaled (common) denominator.
-// optionally (used by GenerateShortestDigits to decide if it has the shortest
-// decimal converting back to v):
-// - v - m-: the distance to the lower boundary.
-// - m+ - v: the distance to the upper boundary.
-//
-// v, m+, m-, and therefore v - m- and m+ - v all share the same denominator.
-//
-// Let ep == estimated_power, then the returned values will satisfy:
-// v / 10^ep = numerator / denominator.
-// v's boundarys m- and m+:
-// m- / 10^ep == v / 10^ep - delta_minus / denominator
-// m+ / 10^ep == v / 10^ep + delta_plus / denominator
-// Or in other words:
-// m- == v - delta_minus * 10^ep / denominator;
-// m+ == v + delta_plus * 10^ep / denominator;
-//
-// Since 10^(k-1) <= v < 10^k (with k == estimated_power)
-// or 10^k <= v < 10^(k+1)
-// we then have 0.1 <= numerator/denominator < 1
-// or 1 <= numerator/denominator < 10
-//
-// It is then easy to kickstart the digit-generation routine.
-//
-// The boundary-deltas are only filled if the mode equals BIGNUM_DTOA_SHORTEST
-// or BIGNUM_DTOA_SHORTEST_SINGLE.
-
-static void InitialScaledStartValues(uint64_t significand,
- int exponent,
- bool lower_boundary_is_closer,
- int estimated_power,
- bool need_boundary_deltas,
- Bignum* numerator,
- Bignum* denominator,
- Bignum* delta_minus,
- Bignum* delta_plus) {
- if (exponent >= 0) {
- InitialScaledStartValuesPositiveExponent(
- significand, exponent, estimated_power, need_boundary_deltas,
- numerator, denominator, delta_minus, delta_plus);
- } else if (estimated_power >= 0) {
- InitialScaledStartValuesNegativeExponentPositivePower(
- significand, exponent, estimated_power, need_boundary_deltas,
- numerator, denominator, delta_minus, delta_plus);
- } else {
- InitialScaledStartValuesNegativeExponentNegativePower(
- significand, exponent, estimated_power, need_boundary_deltas,
- numerator, denominator, delta_minus, delta_plus);
- }
-
- if (need_boundary_deltas && lower_boundary_is_closer) {
- // The lower boundary is closer at half the distance of "normal" numbers.
- // Increase the common denominator and adapt all but the delta_minus.
- denominator->ShiftLeft(1); // *2
- numerator->ShiftLeft(1); // *2
- delta_plus->ShiftLeft(1); // *2
- }
-}
-
-
-// This routine multiplies numerator/denominator so that its values lies in the
-// range 1-10. That is after a call to this function we have:
-// 1 <= (numerator + delta_plus) /denominator < 10.
-// Let numerator the input before modification and numerator' the argument
-// after modification, then the output-parameter decimal_point is such that
-// numerator / denominator * 10^estimated_power ==
-// numerator' / denominator' * 10^(decimal_point - 1)
-// In some cases estimated_power was too low, and this is already the case. We
-// then simply adjust the power so that 10^(k-1) <= v < 10^k (with k ==
-// estimated_power) but do not touch the numerator or denominator.
-// Otherwise the routine multiplies the numerator and the deltas by 10.
-static void FixupMultiply10(int estimated_power, bool is_even,
- int* decimal_point,
- Bignum* numerator, Bignum* denominator,
- Bignum* delta_minus, Bignum* delta_plus) {
- bool in_range;
- if (is_even) {
- // For IEEE doubles half-way cases (in decimal system numbers ending with 5)
- // are rounded to the closest floating-point number with even significand.
- in_range = Bignum::PlusCompare(*numerator, *delta_plus, *denominator) >= 0;
- } else {
- in_range = Bignum::PlusCompare(*numerator, *delta_plus, *denominator) > 0;
- }
- if (in_range) {
- // Since numerator + delta_plus >= denominator we already have
- // 1 <= numerator/denominator < 10. Simply update the estimated_power.
- *decimal_point = estimated_power + 1;
- } else {
- *decimal_point = estimated_power;
- numerator->Times10();
- if (Bignum::Equal(*delta_minus, *delta_plus)) {
- delta_minus->Times10();
- delta_plus->AssignBignum(*delta_minus);
- } else {
- delta_minus->Times10();
- delta_plus->Times10();
- }
- }
-}
-
-} // namespace double_conversion
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/double-conversion/bignum-dtoa.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/double-conversion/bignum-dtoa.h b/ext/kenlm/util/double-conversion/bignum-dtoa.h
deleted file mode 100644
index 34b9619..0000000
--- a/ext/kenlm/util/double-conversion/bignum-dtoa.h
+++ /dev/null
@@ -1,84 +0,0 @@
-// Copyright 2010 the V8 project authors. All rights reserved.
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following
-// disclaimer in the documentation and/or other materials provided
-// with the distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived
-// from this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#ifndef DOUBLE_CONVERSION_BIGNUM_DTOA_H_
-#define DOUBLE_CONVERSION_BIGNUM_DTOA_H_
-
-#include "utils.h"
-
-namespace double_conversion {
-
-enum BignumDtoaMode {
- // Return the shortest correct representation.
- // For example the output of 0.299999999999999988897 is (the less accurate but
- // correct) 0.3.
- BIGNUM_DTOA_SHORTEST,
- // Same as BIGNUM_DTOA_SHORTEST but for single-precision floats.
- BIGNUM_DTOA_SHORTEST_SINGLE,
- // Return a fixed number of digits after the decimal point.
- // For instance fixed(0.1, 4) becomes 0.1000
- // If the input number is big, the output will be big.
- BIGNUM_DTOA_FIXED,
- // Return a fixed number of digits, no matter what the exponent is.
- BIGNUM_DTOA_PRECISION
-};
-
-// Converts the given double 'v' to ascii.
-// The result should be interpreted as buffer * 10^(point-length).
-// The buffer will be null-terminated.
-//
-// The input v must be > 0 and different from NaN, and Infinity.
-//
-// The output depends on the given mode:
-// - SHORTEST: produce the least amount of digits for which the internal
-// identity requirement is still satisfied. If the digits are printed
-// (together with the correct exponent) then reading this number will give
-// 'v' again. The buffer will choose the representation that is closest to
-// 'v'. If there are two at the same distance, than the number is round up.
-// In this mode the 'requested_digits' parameter is ignored.
-// - FIXED: produces digits necessary to print a given number with
-// 'requested_digits' digits after the decimal point. The produced digits
-// might be too short in which case the caller has to fill the gaps with '0's.
-// Example: toFixed(0.001, 5) is allowed to return buffer="1", point=-2.
-// Halfway cases are rounded up. The call toFixed(0.15, 2) thus returns
-// buffer="2", point=0.
-// Note: the length of the returned buffer has no meaning wrt the significance
-// of its digits. That is, just because it contains '0's does not mean that
-// any other digit would not satisfy the internal identity requirement.
-// - PRECISION: produces 'requested_digits' where the first digit is not '0'.
-// Even though the length of produced digits usually equals
-// 'requested_digits', the function is allowed to return fewer digits, in
-// which case the caller has to fill the missing digits with '0's.
-// Halfway cases are again rounded up.
-// 'BignumDtoa' expects the given buffer to be big enough to hold all digits
-// and a terminating null-character.
-void BignumDtoa(double v, BignumDtoaMode mode, int requested_digits,
- Vector<char> buffer, int* length, int* point);
-
-} // namespace double_conversion
-
-#endif // DOUBLE_CONVERSION_BIGNUM_DTOA_H_
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/util/double-conversion/bignum.cc
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/util/double-conversion/bignum.cc b/ext/kenlm/util/double-conversion/bignum.cc
deleted file mode 100644
index 747491a..0000000
--- a/ext/kenlm/util/double-conversion/bignum.cc
+++ /dev/null
@@ -1,764 +0,0 @@
-// Copyright 2010 the V8 project authors. All rights reserved.
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following
-// disclaimer in the documentation and/or other materials provided
-// with the distribution.
-// * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived
-// from this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#include "bignum.h"
-#include "utils.h"
-
-namespace double_conversion {
-
-Bignum::Bignum()
- : bigits_(bigits_buffer_, kBigitCapacity), used_digits_(0), exponent_(0) {
- for (int i = 0; i < kBigitCapacity; ++i) {
- bigits_[i] = 0;
- }
-}
-
-
-template<typename S>
-static int BitSize(S value) {
- return 8 * sizeof(value);
-}
-
-// Guaranteed to lie in one Bigit.
-void Bignum::AssignUInt16(uint16_t value) {
- ASSERT(kBigitSize >= BitSize(value));
- Zero();
- if (value == 0) return;
-
- EnsureCapacity(1);
- bigits_[0] = value;
- used_digits_ = 1;
-}
-
-
-void Bignum::AssignUInt64(uint64_t value) {
- const int kUInt64Size = 64;
-
- Zero();
- if (value == 0) return;
-
- int needed_bigits = kUInt64Size / kBigitSize + 1;
- EnsureCapacity(needed_bigits);
- for (int i = 0; i < needed_bigits; ++i) {
- bigits_[i] = value & kBigitMask;
- value = value >> kBigitSize;
- }
- used_digits_ = needed_bigits;
- Clamp();
-}
-
-
-void Bignum::AssignBignum(const Bignum& other) {
- exponent_ = other.exponent_;
- for (int i = 0; i < other.used_digits_; ++i) {
- bigits_[i] = other.bigits_[i];
- }
- // Clear the excess digits (if there were any).
- for (int i = other.used_digits_; i < used_digits_; ++i) {
- bigits_[i] = 0;
- }
- used_digits_ = other.used_digits_;
-}
-
-
-static uint64_t ReadUInt64(Vector<const char> buffer,
- int from,
- int digits_to_read) {
- uint64_t result = 0;
- for (int i = from; i < from + digits_to_read; ++i) {
- int digit = buffer[i] - '0';
- ASSERT(0 <= digit && digit <= 9);
- result = result * 10 + digit;
- }
- return result;
-}
-
-
-void Bignum::AssignDecimalString(Vector<const char> value) {
- // 2^64 = 18446744073709551616 > 10^19
- const int kMaxUint64DecimalDigits = 19;
- Zero();
- int length = value.length();
- int pos = 0;
- // Let's just say that each digit needs 4 bits.
- while (length >= kMaxUint64DecimalDigits) {
- uint64_t digits = ReadUInt64(value, pos, kMaxUint64DecimalDigits);
- pos += kMaxUint64DecimalDigits;
- length -= kMaxUint64DecimalDigits;
- MultiplyByPowerOfTen(kMaxUint64DecimalDigits);
- AddUInt64(digits);
- }
- uint64_t digits = ReadUInt64(value, pos, length);
- MultiplyByPowerOfTen(length);
- AddUInt64(digits);
- Clamp();
-}
-
-
-static int HexCharValue(char c) {
- if ('0' <= c && c <= '9') return c - '0';
- if ('a' <= c && c <= 'f') return 10 + c - 'a';
- if ('A' <= c && c <= 'F') return 10 + c - 'A';
- UNREACHABLE();
- return 0; // To make compiler happy.
-}
-
-
-void Bignum::AssignHexString(Vector<const char> value) {
- Zero();
- int length = value.length();
-
- int needed_bigits = length * 4 / kBigitSize + 1;
- EnsureCapacity(needed_bigits);
- int string_index = length - 1;
- for (int i = 0; i < needed_bigits - 1; ++i) {
- // These bigits are guaranteed to be "full".
- Chunk current_bigit = 0;
- for (int j = 0; j < kBigitSize / 4; j++) {
- current_bigit += HexCharValue(value[string_index--]) << (j * 4);
- }
- bigits_[i] = current_bigit;
- }
- used_digits_ = needed_bigits - 1;
-
- Chunk most_significant_bigit = 0; // Could be = 0;
- for (int j = 0; j <= string_index; ++j) {
- most_significant_bigit <<= 4;
- most_significant_bigit += HexCharValue(value[j]);
- }
- if (most_significant_bigit != 0) {
- bigits_[used_digits_] = most_significant_bigit;
- used_digits_++;
- }
- Clamp();
-}
-
-
-void Bignum::AddUInt64(uint64_t operand) {
- if (operand == 0) return;
- Bignum other;
- other.AssignUInt64(operand);
- AddBignum(other);
-}
-
-
-void Bignum::AddBignum(const Bignum& other) {
- ASSERT(IsClamped());
- ASSERT(other.IsClamped());
-
- // If this has a greater exponent than other append zero-bigits to this.
- // After this call exponent_ <= other.exponent_.
- Align(other);
-
- // There are two possibilities:
- // aaaaaaaaaaa 0000 (where the 0s represent a's exponent)
- // bbbbb 00000000
- // ----------------
- // ccccccccccc 0000
- // or
- // aaaaaaaaaa 0000
- // bbbbbbbbb 0000000
- // -----------------
- // cccccccccccc 0000
- // In both cases we might need a carry bigit.
-
- EnsureCapacity(1 + Max(BigitLength(), other.BigitLength()) - exponent_);
- Chunk carry = 0;
- int bigit_pos = other.exponent_ - exponent_;
- ASSERT(bigit_pos >= 0);
- for (int i = 0; i < other.used_digits_; ++i) {
- Chunk sum = bigits_[bigit_pos] + other.bigits_[i] + carry;
- bigits_[bigit_pos] = sum & kBigitMask;
- carry = sum >> kBigitSize;
- bigit_pos++;
- }
-
- while (carry != 0) {
- Chunk sum = bigits_[bigit_pos] + carry;
- bigits_[bigit_pos] = sum & kBigitMask;
- carry = sum >> kBigitSize;
- bigit_pos++;
- }
- used_digits_ = Max(bigit_pos, used_digits_);
- ASSERT(IsClamped());
-}
-
-
-void Bignum::SubtractBignum(const Bignum& other) {
- ASSERT(IsClamped());
- ASSERT(other.IsClamped());
- // We require this to be bigger than other.
- ASSERT(LessEqual(other, *this));
-
- Align(other);
-
- int offset = other.exponent_ - exponent_;
- Chunk borrow = 0;
- int i;
- for (i = 0; i < other.used_digits_; ++i) {
- ASSERT((borrow == 0) || (borrow == 1));
- Chunk difference = bigits_[i + offset] - other.bigits_[i] - borrow;
- bigits_[i + offset] = difference & kBigitMask;
- borrow = difference >> (kChunkSize - 1);
- }
- while (borrow != 0) {
- Chunk difference = bigits_[i + offset] - borrow;
- bigits_[i + offset] = difference & kBigitMask;
- borrow = difference >> (kChunkSize - 1);
- ++i;
- }
- Clamp();
-}
-
-
-void Bignum::ShiftLeft(int shift_amount) {
- if (used_digits_ == 0) return;
- exponent_ += shift_amount / kBigitSize;
- int local_shift = shift_amount % kBigitSize;
- EnsureCapacity(used_digits_ + 1);
- BigitsShiftLeft(local_shift);
-}
-
-
-void Bignum::MultiplyByUInt32(uint32_t factor) {
- if (factor == 1) return;
- if (factor == 0) {
- Zero();
- return;
- }
- if (used_digits_ == 0) return;
-
- // The product of a bigit with the factor is of size kBigitSize + 32.
- // Assert that this number + 1 (for the carry) fits into double chunk.
- ASSERT(kDoubleChunkSize >= kBigitSize + 32 + 1);
- DoubleChunk carry = 0;
- for (int i = 0; i < used_digits_; ++i) {
- DoubleChunk product = static_cast<DoubleChunk>(factor) * bigits_[i] + carry;
- bigits_[i] = static_cast<Chunk>(product & kBigitMask);
- carry = (product >> kBigitSize);
- }
- while (carry != 0) {
- EnsureCapacity(used_digits_ + 1);
- bigits_[used_digits_] = carry & kBigitMask;
- used_digits_++;
- carry >>= kBigitSize;
- }
-}
-
-
-void Bignum::MultiplyByUInt64(uint64_t factor) {
- if (factor == 1) return;
- if (factor == 0) {
- Zero();
- return;
- }
- ASSERT(kBigitSize < 32);
- uint64_t carry = 0;
- uint64_t low = factor & 0xFFFFFFFF;
- uint64_t high = factor >> 32;
- for (int i = 0; i < used_digits_; ++i) {
- uint64_t product_low = low * bigits_[i];
- uint64_t product_high = high * bigits_[i];
- uint64_t tmp = (carry & kBigitMask) + product_low;
- bigits_[i] = tmp & kBigitMask;
- carry = (carry >> kBigitSize) + (tmp >> kBigitSize) +
- (product_high << (32 - kBigitSize));
- }
- while (carry != 0) {
- EnsureCapacity(used_digits_ + 1);
- bigits_[used_digits_] = carry & kBigitMask;
- used_digits_++;
- carry >>= kBigitSize;
- }
-}
-
-
-void Bignum::MultiplyByPowerOfTen(int exponent) {
- const uint64_t kFive27 = UINT64_2PART_C(0x6765c793, fa10079d);
- const uint16_t kFive1 = 5;
- const uint16_t kFive2 = kFive1 * 5;
- const uint16_t kFive3 = kFive2 * 5;
- const uint16_t kFive4 = kFive3 * 5;
- const uint16_t kFive5 = kFive4 * 5;
- const uint16_t kFive6 = kFive5 * 5;
- const uint32_t kFive7 = kFive6 * 5;
- const uint32_t kFive8 = kFive7 * 5;
- const uint32_t kFive9 = kFive8 * 5;
- const uint32_t kFive10 = kFive9 * 5;
- const uint32_t kFive11 = kFive10 * 5;
- const uint32_t kFive12 = kFive11 * 5;
- const uint32_t kFive13 = kFive12 * 5;
- const uint32_t kFive1_to_12[] =
- { kFive1, kFive2, kFive3, kFive4, kFive5, kFive6,
- kFive7, kFive8, kFive9, kFive10, kFive11, kFive12 };
-
- ASSERT(exponent >= 0);
- if (exponent == 0) return;
- if (used_digits_ == 0) return;
-
- // We shift by exponent at the end just before returning.
- int remaining_exponent = exponent;
- while (remaining_exponent >= 27) {
- MultiplyByUInt64(kFive27);
- remaining_exponent -= 27;
- }
- while (remaining_exponent >= 13) {
- MultiplyByUInt32(kFive13);
- remaining_exponent -= 13;
- }
- if (remaining_exponent > 0) {
- MultiplyByUInt32(kFive1_to_12[remaining_exponent - 1]);
- }
- ShiftLeft(exponent);
-}
-
-
-void Bignum::Square() {
- ASSERT(IsClamped());
- int product_length = 2 * used_digits_;
- EnsureCapacity(product_length);
-
- // Comba multiplication: compute each column separately.
- // Example: r = a2a1a0 * b2b1b0.
- // r = 1 * a0b0 +
- // 10 * (a1b0 + a0b1) +
- // 100 * (a2b0 + a1b1 + a0b2) +
- // 1000 * (a2b1 + a1b2) +
- // 10000 * a2b2
- //
- // In the worst case we have to accumulate nb-digits products of digit*digit.
- //
- // Assert that the additional number of bits in a DoubleChunk are enough to
- // sum up used_digits of Bigit*Bigit.
- if ((1 << (2 * (kChunkSize - kBigitSize))) <= used_digits_) {
- UNIMPLEMENTED();
- }
- DoubleChunk accumulator = 0;
- // First shift the digits so we don't overwrite them.
- int copy_offset = used_digits_;
- for (int i = 0; i < used_digits_; ++i) {
- bigits_[copy_offset + i] = bigits_[i];
- }
- // We have two loops to avoid some 'if's in the loop.
- for (int i = 0; i < used_digits_; ++i) {
- // Process temporary digit i with power i.
- // The sum of the two indices must be equal to i.
- int bigit_index1 = i;
- int bigit_index2 = 0;
- // Sum all of the sub-products.
- while (bigit_index1 >= 0) {
- Chunk chunk1 = bigits_[copy_offset + bigit_index1];
- Chunk chunk2 = bigits_[copy_offset + bigit_index2];
- accumulator += static_cast<DoubleChunk>(chunk1) * chunk2;
- bigit_index1--;
- bigit_index2++;
- }
- bigits_[i] = static_cast<Chunk>(accumulator) & kBigitMask;
- accumulator >>= kBigitSize;
- }
- for (int i = used_digits_; i < product_length; ++i) {
- int bigit_index1 = used_digits_ - 1;
- int bigit_index2 = i - bigit_index1;
- // Invariant: sum of both indices is again equal to i.
- // Inner loop runs 0 times on last iteration, emptying accumulator.
- while (bigit_index2 < used_digits_) {
- Chunk chunk1 = bigits_[copy_offset + bigit_index1];
- Chunk chunk2 = bigits_[copy_offset + bigit_index2];
- accumulator += static_cast<DoubleChunk>(chunk1) * chunk2;
- bigit_index1--;
- bigit_index2++;
- }
- // The overwritten bigits_[i] will never be read in further loop iterations,
- // because bigit_index1 and bigit_index2 are always greater
- // than i - used_digits_.
- bigits_[i] = static_cast<Chunk>(accumulator) & kBigitMask;
- accumulator >>= kBigitSize;
- }
- // Since the result was guaranteed to lie inside the number the
- // accumulator must be 0 now.
- ASSERT(accumulator == 0);
-
- // Don't forget to update the used_digits and the exponent.
- used_digits_ = product_length;
- exponent_ *= 2;
- Clamp();
-}
-
-
-void Bignum::AssignPowerUInt16(uint16_t base, int power_exponent) {
- ASSERT(base != 0);
- ASSERT(power_exponent >= 0);
- if (power_exponent == 0) {
- AssignUInt16(1);
- return;
- }
- Zero();
- int shifts = 0;
- // We expect base to be in range 2-32, and most often to be 10.
- // It does not make much sense to implement different algorithms for counting
- // the bits.
- while ((base & 1) == 0) {
- base >>= 1;
- shifts++;
- }
- int bit_size = 0;
- int tmp_base = base;
- while (tmp_base != 0) {
- tmp_base >>= 1;
- bit_size++;
- }
- int final_size = bit_size * power_exponent;
- // 1 extra bigit for the shifting, and one for rounded final_size.
- EnsureCapacity(final_size / kBigitSize + 2);
-
- // Left to Right exponentiation.
- int mask = 1;
- while (power_exponent >= mask) mask <<= 1;
-
- // The mask is now pointing to the bit above the most significant 1-bit of
- // power_exponent.
- // Get rid of first 1-bit;
- mask >>= 2;
- uint64_t this_value = base;
-
- bool delayed_multipliciation = false;
- const uint64_t max_32bits = 0xFFFFFFFF;
- while (mask != 0 && this_value <= max_32bits) {
- this_value = this_value * this_value;
- // Verify that there is enough space in this_value to perform the
- // multiplication. The first bit_size bits must be 0.
- if ((power_exponent & mask) != 0) {
- uint64_t base_bits_mask =
- ~((static_cast<uint64_t>(1) << (64 - bit_size)) - 1);
- bool high_bits_zero = (this_value & base_bits_mask) == 0;
- if (high_bits_zero) {
- this_value *= base;
- } else {
- delayed_multipliciation = true;
- }
- }
- mask >>= 1;
- }
- AssignUInt64(this_value);
- if (delayed_multipliciation) {
- MultiplyByUInt32(base);
- }
-
- // Now do the same thing as a bignum.
- while (mask != 0) {
- Square();
- if ((power_exponent & mask) != 0) {
- MultiplyByUInt32(base);
- }
- mask >>= 1;
- }
-
- // And finally add the saved shifts.
- ShiftLeft(shifts * power_exponent);
-}
-
-
-// Precondition: this/other < 16bit.
-uint16_t Bignum::DivideModuloIntBignum(const Bignum& other) {
- ASSERT(IsClamped());
- ASSERT(other.IsClamped());
- ASSERT(other.used_digits_ > 0);
-
- // Easy case: if we have less digits than the divisor than the result is 0.
- // Note: this handles the case where this == 0, too.
- if (BigitLength() < other.BigitLength()) {
- return 0;
- }
-
- Align(other);
-
- uint16_t result = 0;
-
- // Start by removing multiples of 'other' until both numbers have the same
- // number of digits.
- while (BigitLength() > other.BigitLength()) {
- // This naive approach is extremely inefficient if the this divided other
- // might be big. This function is implemented for doubleToString where
- // the result should be small (less than 10).
- ASSERT(other.bigits_[other.used_digits_ - 1] >= ((1 << kBigitSize) / 16));
- // Remove the multiples of the first digit.
- // Example this = 23 and other equals 9. -> Remove 2 multiples.
- result += bigits_[used_digits_ - 1];
- SubtractTimes(other, bigits_[used_digits_ - 1]);
- }
-
- ASSERT(BigitLength() == other.BigitLength());
-
- // Both bignums are at the same length now.
- // Since other has more than 0 digits we know that the access to
- // bigits_[used_digits_ - 1] is safe.
- Chunk this_bigit = bigits_[used_digits_ - 1];
- Chunk other_bigit = other.bigits_[other.used_digits_ - 1];
-
- if (other.used_digits_ == 1) {
- // Shortcut for easy (and common) case.
- int quotient = this_bigit / other_bigit;
- bigits_[used_digits_ - 1] = this_bigit - other_bigit * quotient;
- result += quotient;
- Clamp();
- return result;
- }
-
- int division_estimate = this_bigit / (other_bigit + 1);
- result += division_estimate;
- SubtractTimes(other, division_estimate);
-
- if (other_bigit * (division_estimate + 1) > this_bigit) {
- // No need to even try to subtract. Even if other's remaining digits were 0
- // another subtraction would be too much.
- return result;
- }
-
- while (LessEqual(other, *this)) {
- SubtractBignum(other);
- result++;
- }
- return result;
-}
-
-
-template<typename S>
-static int SizeInHexChars(S number) {
- ASSERT(number > 0);
- int result = 0;
- while (number != 0) {
- number >>= 4;
- result++;
- }
- return result;
-}
-
-
-static char HexCharOfValue(int value) {
- ASSERT(0 <= value && value <= 16);
- if (value < 10) return value + '0';
- return value - 10 + 'A';
-}
-
-
-bool Bignum::ToHexString(char* buffer, int buffer_size) const {
- ASSERT(IsClamped());
- // Each bigit must be printable as separate hex-character.
- ASSERT(kBigitSize % 4 == 0);
- const int kHexCharsPerBigit = kBigitSize / 4;
-
- if (used_digits_ == 0) {
- if (buffer_size < 2) return false;
- buffer[0] = '0';
- buffer[1] = '\0';
- return true;
- }
- // We add 1 for the terminating '\0' character.
- int needed_chars = (BigitLength() - 1) * kHexCharsPerBigit +
- SizeInHexChars(bigits_[used_digits_ - 1]) + 1;
- if (needed_chars > buffer_size) return false;
- int string_index = needed_chars - 1;
- buffer[string_index--] = '\0';
- for (int i = 0; i < exponent_; ++i) {
- for (int j = 0; j < kHexCharsPerBigit; ++j) {
- buffer[string_index--] = '0';
- }
- }
- for (int i = 0; i < used_digits_ - 1; ++i) {
- Chunk current_bigit = bigits_[i];
- for (int j = 0; j < kHexCharsPerBigit; ++j) {
- buffer[string_index--] = HexCharOfValue(current_bigit & 0xF);
- current_bigit >>= 4;
- }
- }
- // And finally the last bigit.
- Chunk most_significant_bigit = bigits_[used_digits_ - 1];
- while (most_significant_bigit != 0) {
- buffer[string_index--] = HexCharOfValue(most_significant_bigit & 0xF);
- most_significant_bigit >>= 4;
- }
- return true;
-}
-
-
-Bignum::Chunk Bignum::BigitAt(int index) const {
- if (index >= BigitLength()) return 0;
- if (index < exponent_) return 0;
- return bigits_[index - exponent_];
-}
-
-
-int Bignum::Compare(const Bignum& a, const Bignum& b) {
- ASSERT(a.IsClamped());
- ASSERT(b.IsClamped());
- int bigit_length_a = a.BigitLength();
- int bigit_length_b = b.BigitLength();
- if (bigit_length_a < bigit_length_b) return -1;
- if (bigit_length_a > bigit_length_b) return +1;
- for (int i = bigit_length_a - 1; i >= Min(a.exponent_, b.exponent_); --i) {
- Chunk bigit_a = a.BigitAt(i);
- Chunk bigit_b = b.BigitAt(i);
- if (bigit_a < bigit_b) return -1;
- if (bigit_a > bigit_b) return +1;
- // Otherwise they are equal up to this digit. Try the next digit.
- }
- return 0;
-}
-
-
-int Bignum::PlusCompare(const Bignum& a, const Bignum& b, const Bignum& c) {
- ASSERT(a.IsClamped());
- ASSERT(b.IsClamped());
- ASSERT(c.IsClamped());
- if (a.BigitLength() < b.BigitLength()) {
- return PlusCompare(b, a, c);
- }
- if (a.BigitLength() + 1 < c.BigitLength()) return -1;
- if (a.BigitLength() > c.BigitLength()) return +1;
- // The exponent encodes 0-bigits. So if there are more 0-digits in 'a' than
- // 'b' has digits, then the bigit-length of 'a'+'b' must be equal to the one
- // of 'a'.
- if (a.exponent_ >= b.BigitLength() && a.BigitLength() < c.BigitLength()) {
- return -1;
- }
-
- Chunk borrow = 0;
- // Starting at min_exponent all digits are == 0. So no need to compare them.
- int min_exponent = Min(Min(a.exponent_, b.exponent_), c.exponent_);
- for (int i = c.BigitLength() - 1; i >= min_exponent; --i) {
- Chunk chunk_a = a.BigitAt(i);
- Chunk chunk_b = b.BigitAt(i);
- Chunk chunk_c = c.BigitAt(i);
- Chunk sum = chunk_a + chunk_b;
- if (sum > chunk_c + borrow) {
- return +1;
- } else {
- borrow = chunk_c + borrow - sum;
- if (borrow > 1) return -1;
- borrow <<= kBigitSize;
- }
- }
- if (borrow == 0) return 0;
- return -1;
-}
-
-
-void Bignum::Clamp() {
- while (used_digits_ > 0 && bigits_[used_digits_ - 1] == 0) {
- used_digits_--;
- }
- if (used_digits_ == 0) {
- // Zero.
- exponent_ = 0;
- }
-}
-
-
-bool Bignum::IsClamped() const {
- return used_digits_ == 0 || bigits_[used_digits_ - 1] != 0;
-}
-
-
-void Bignum::Zero() {
- for (int i = 0; i < used_digits_; ++i) {
- bigits_[i] = 0;
- }
- used_digits_ = 0;
- exponent_ = 0;
-}
-
-
-void Bignum::Align(const Bignum& other) {
- if (exponent_ > other.exponent_) {
- // If "X" represents a "hidden" digit (by the exponent) then we are in the
- // following case (a == this, b == other):
- // a: aaaaaaXXXX or a: aaaaaXXX
- // b: bbbbbbX b: bbbbbbbbXX
- // We replace some of the hidden digits (X) of a with 0 digits.
- // a: aaaaaa000X or a: aaaaa0XX
- int zero_digits = exponent_ - other.exponent_;
- EnsureCapacity(used_digits_ + zero_digits);
- for (int i = used_digits_ - 1; i >= 0; --i) {
- bigits_[i + zero_digits] = bigits_[i];
- }
- for (int i = 0; i < zero_digits; ++i) {
- bigits_[i] = 0;
- }
- used_digits_ += zero_digits;
- exponent_ -= zero_digits;
- ASSERT(used_digits_ >= 0);
- ASSERT(exponent_ >= 0);
- }
-}
-
-
-void Bignum::BigitsShiftLeft(int shift_amount) {
- ASSERT(shift_amount < kBigitSize);
- ASSERT(shift_amount >= 0);
- Chunk carry = 0;
- for (int i = 0; i < used_digits_; ++i) {
- Chunk new_carry = bigits_[i] >> (kBigitSize - shift_amount);
- bigits_[i] = ((bigits_[i] << shift_amount) + carry) & kBigitMask;
- carry = new_carry;
- }
- if (carry != 0) {
- bigits_[used_digits_] = carry;
- used_digits_++;
- }
-}
-
-
-void Bignum::SubtractTimes(const Bignum& other, int factor) {
- ASSERT(exponent_ <= other.exponent_);
- if (factor < 3) {
- for (int i = 0; i < factor; ++i) {
- SubtractBignum(other);
- }
- return;
- }
- Chunk borrow = 0;
- int exponent_diff = other.exponent_ - exponent_;
- for (int i = 0; i < other.used_digits_; ++i) {
- DoubleChunk product = static_cast<DoubleChunk>(factor) * other.bigits_[i];
- DoubleChunk remove = borrow + product;
- Chunk difference = bigits_[i + exponent_diff] - (remove & kBigitMask);
- bigits_[i + exponent_diff] = difference & kBigitMask;
- borrow = static_cast<Chunk>((difference >> (kChunkSize - 1)) +
- (remove >> kBigitSize));
- }
- for (int i = other.used_digits_ + exponent_diff; i < used_digits_; ++i) {
- if (borrow == 0) return;
- Chunk difference = bigits_[i] - borrow;
- bigits_[i] = difference & kBigitMask;
- borrow = difference >> (kChunkSize - 1);
- ++i;
- }
- Clamp();
-}
-
-
-} // namespace double_conversion
[39/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/fortran.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/fortran.jam b/ext/kenlm/jam-files/boost-build/tools/fortran.jam
deleted file mode 100644
index 3766582..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/fortran.jam
+++ /dev/null
@@ -1,55 +0,0 @@
-# Copyright (C) 2004 Toon Knapen
-#
-# Use, modification and distribution is subject to the Boost Software
-# License Version 1.0. (See accompanying file LICENSE_1_0.txt or
-# http://www.boost.org/LICENSE_1_0.txt)
-
-#
-# This file contains common settings for all fortran tools
-#
-
-import "class" : new ;
-import feature : feature ;
-
-import type ;
-import generators ;
-import common ;
-
-type.register FORTRAN : f F for f77 ;
-type.register FORTRAN90 : f90 F90 ;
-
-feature fortran : : free ;
-feature fortran90 : : free ;
-
-class fortran-compiling-generator : generator
-{
- rule __init__ ( id : source-types + : target-types + : requirements * : optional-properties * )
- {
- generator.__init__ $(id) : $(source-types) : $(target-types) : $(requirements) : $(optional-properties) ;
- }
-}
-
-rule register-fortran-compiler ( id : source-types + : target-types + : requirements * : optional-properties * )
-{
- local g = [ new fortran-compiling-generator $(id) : $(source-types) : $(target-types) : $(requirements) : $(optional-properties) ] ;
- generators.register $(g) ;
-}
-
-class fortran90-compiling-generator : generator
-{
- rule __init__ ( id : source-types + : target-types + : requirements * : optional-properties * )
- {
- generator.__init__ $(id) : $(source-types) : $(target-types) : $(requirements) : $(optional-properties) ;
- }
-}
-
-rule register-fortran90-compiler ( id : source-types + : target-types + : requirements * : optional-properties * )
-{
- local g = [ new fortran90-compiling-generator $(id) : $(source-types) : $(target-types) : $(requirements) : $(optional-properties) ] ;
- generators.register $(g) ;
-}
-
-# FIXME: this is ugly, should find a better way (we'd want client code to
-# register all generators as "generator.some-rule", not with "some-module.some-rule".)
-IMPORT $(__name__) : register-fortran-compiler : : generators.register-fortran-compiler ;
-IMPORT $(__name__) : register-fortran90-compiler : : generators.register-fortran90-compiler ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/gcc.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/gcc.jam b/ext/kenlm/jam-files/boost-build/tools/gcc.jam
deleted file mode 100644
index ef90f05..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/gcc.jam
+++ /dev/null
@@ -1,1184 +0,0 @@
-# Copyright 2001 David Abrahams
-# Copyright 2002-2006 Rene Rivera
-# Copyright 2002-2003 Vladimir Prus
-# Copyright 2005 Reece H. Dunn
-# Copyright 2006 Ilya Sokolov
-# Copyright 2007 Roland Schwarz
-# Copyright 2007 Boris Gubenko
-#
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-import "class" : new ;
-import common ;
-import feature ;
-import fortran ;
-import generators ;
-import os ;
-import pch ;
-import property ;
-import property-set ;
-import rc ;
-import regex ;
-import set ;
-import toolset ;
-import type ;
-import unix ;
-
-
-if [ MATCH (--debug-configuration) : [ modules.peek : ARGV ] ]
-{
- .debug-configuration = true ;
-}
-
-
-feature.extend toolset : gcc ;
-# feature.subfeature toolset gcc : flavor : : optional ;
-
-toolset.inherit-generators gcc : unix : unix.link unix.link.dll ;
-toolset.inherit-flags gcc : unix ;
-toolset.inherit-rules gcc : unix ;
-
-generators.override gcc.prebuilt : builtin.prebuilt ;
-generators.override gcc.searched-lib-generator : searched-lib-generator ;
-
-# Make gcc toolset object files use the "o" suffix on all platforms.
-type.set-generated-target-suffix OBJ : <toolset>gcc : o ;
-type.set-generated-target-suffix OBJ : <toolset>gcc <target-os>windows : o ;
-type.set-generated-target-suffix OBJ : <toolset>gcc <target-os>cygwin : o ;
-
-
-# Initializes the gcc toolset for the given version. If necessary, command may
-# be used to specify where the compiler is located. The parameter 'options' is a
-# space-delimited list of options, each one specified as
-# <option-name>option-value. Valid option names are: cxxflags, linkflags and
-# linker-type. Accepted linker-type values are aix, darwin, gnu, hpux, osf or
-# sun and the default value will be selected based on the current OS.
-# Example:
-# using gcc : 3.4 : : <cxxflags>foo <linkflags>bar <linker-type>sun ;
-#
-# The compiler command to use is detected in three steps:
-# 1) If an explicit command is specified by the user, it will be used and must
-# be available.
-# 2) If only a certain version is specified, it is enforced:
-# - either the 'g++-VERSION' command must be available
-# - or the default command 'g++' must be available and match the exact
-# version.
-# 3) Without user-provided restrictions use default 'g++'.
-#
-rule init ( version ? : command * : options * )
-{
- #1): use user-provided command
- local tool-command = ;
- if $(command)
- {
- tool-command = [ common.get-invocation-command-nodefault gcc : g++ :
- $(command) ] ;
- if ! $(tool-command)
- {
- import errors ;
- errors.error toolset gcc initialization:
- : provided command '$(command)' not found
- : initialized from [ errors.nearest-user-location ] ;
- }
- }
- #2): enforce user-provided version
- else if $(version)
- {
- tool-command = [ common.get-invocation-command-nodefault gcc :
- "g++-$(version[1])" ] ;
-
- #2.1) fallback: check whether "g++" reports the requested version
- if ! $(tool-command)
- {
- tool-command = [ common.get-invocation-command-nodefault gcc : g++ ]
- ;
- if $(tool-command)
- {
- local tool-command-string = $(tool-command:J=" ") ;
- local tool-version = [ MATCH "^([0-9.]+)" :
- [ SHELL "$(tool-command-string) -dumpversion" ] ] ;
- if $(tool-version) != $(version)
- {
- # Permit a match betwen a two-digit version specified by the
- # user (e.g. 4.4) and a 3-digit version reported by gcc.
- # Since only two digits are present in the binary name
- # anyway, insisting that user specify the 3-digit version
- # when configuring Boost.Build, while it is not required on
- # the command line, would be strange.
- local stripped = [ MATCH "^([0-9]+\.[0-9]+).*" :
- $(tool-version) ] ;
- if $(stripped) != $(version)
- {
- import errors ;
- errors.error toolset gcc initialization:
- : version '$(version)' requested but
- 'g++-$(version)' not found and version
- '$(tool-version)' of default '$(tool-command)'
- does not match
- : initialized from [ errors.nearest-user-location ]
- ;
- tool-command = ;
- }
- # Use full 3-digit version to be compatible with the
- # 'using gcc ;' case
- version = $(tool-version) ;
- }
- }
- else
- {
- import errors ;
- errors.error toolset gcc initialization:
- : version '$(version)' requested but neither
- 'g++-$(version)' nor default 'g++' found
- : initialized from [ errors.nearest-user-location ] ;
- }
- }
- }
- #3) default: no command and no version specified, try using "g++"
- else
- {
- tool-command = [ common.get-invocation-command-nodefault gcc : g++ ] ;
- if ! $(tool-command)
- {
- import errors ;
- errors.error toolset gcc initialization:
- : no command provided, default command 'g++' not found
- : initialized from [ errors.nearest-user-location ] ;
- }
- }
-
-
- # Information about the gcc command...
- # The command.
- local command = $(tool-command) ;
- # The 'command' variable can have multiple elements but when calling the
- # SHELL builtin we need a single string.
- local command-string = $(command:J=" ") ;
- # The root directory of the tool install.
- local root = [ feature.get-values <root> : $(options) ] ;
- # The bin directory where to find the command to execute.
- local bin ;
- # The compiler flavor.
- local flavor = [ feature.get-values <flavor> : $(options) ] ;
- # Autodetect the root and bin dir if not given.
- if $(command)
- {
- bin ?= [ common.get-absolute-tool-path $(command[-1]) ] ;
- root ?= $(bin:D) ;
- }
- # Autodetect the version and flavor if not given.
- if $(command)
- {
- local machine = [ MATCH "^([^ ]+)" :
- [ SHELL "$(command-string) -dumpmachine" ] ] ;
- version ?= [ MATCH "^([0-9.]+)" :
- [ SHELL "$(command-string) -dumpversion" ] ] ;
- switch $(machine:L)
- {
- case *mingw* : flavor ?= mingw ;
- }
- }
-
- local condition ;
- if $(flavor)
- {
- condition = flavor $(flavor) ;
- }
- condition = [ common.check-init-parameters gcc : version $(version)
- : $(condition) ] ;
-
- common.handle-options gcc : $(condition) : $(command) : $(options) ;
-
- local linker = [ feature.get-values <linker-type> : $(options) ] ;
- # TODO: The logic below should actually be keyed on <target-os>.
- if ! $(linker)
- {
- switch [ os.name ]
- {
- case OSF : linker = osf ;
- case HPUX : linker = hpux ;
- case AIX : linker = aix ;
- case SOLARIS : linker = sun ;
- case * : linker = gnu ;
- }
- }
- init-link-flags gcc $(linker) $(condition) ;
-
- # If gcc is installed in a non-standard location, we would need to add
- # LD_LIBRARY_PATH when running programs created with it (for unit-test/run
- # rules).
- if $(command)
- {
- # On multilib 64-bit boxes, there are both 32-bit and 64-bit libraries
- # and all must be added to LD_LIBRARY_PATH. The linker will pick the
- # right onces. Note that we do not provide a clean way to build a 32-bit
- # binary using a 64-bit compiler, but user can always pass -m32
- # manually.
- local lib_path = $(root)/bin $(root)/lib $(root)/lib32 $(root)/lib64 ;
- if $(.debug-configuration)
- {
- ECHO notice: using gcc libraries :: $(condition) :: $(lib_path) ;
- }
- toolset.flags gcc.link RUN_PATH $(condition) : $(lib_path) ;
- }
-
- # If we are not using a system gcc installation we should adjust the various
- # programs as needed to prefer using their installation specific versions.
- # This is essential for correct use of MinGW and for cross-compiling.
-
- local nl = "
-" ;
-
- # - Archive builder.
- local archiver = [ common.get-invocation-command gcc
- : [ NORMALIZE_PATH [ MATCH "(.*)[$(nl)]+" :
- [ SHELL "$(command-string) -print-prog-name=ar" ] ] ]
- : [ feature.get-values <archiver> : $(options) ]
- : $(bin)
- : search-path ] ;
- toolset.flags gcc.archive .AR $(condition) : $(archiver[1]) ;
- if $(.debug-configuration)
- {
- ECHO notice: using gcc archiver :: $(condition) :: $(archiver[1]) ;
- }
-
- # - Ranlib.
- local ranlib = [ common.get-invocation-command gcc
- : [ NORMALIZE_PATH [ MATCH "(.*)[$(nl)]+" :
- [ SHELL "$(command-string) -print-prog-name=ranlib" ] ] ]
- : [ feature.get-values <ranlib> : $(options) ]
- : $(bin)
- : search-path ] ;
- toolset.flags gcc.archive .RANLIB $(condition) : $(ranlib[1]) ;
- if $(.debug-configuration)
- {
- ECHO notice: using gcc ranlib :: $(condition) :: $(ranlib[1]) ;
- }
-
- # - Resource compiler.
- local rc = [ common.get-invocation-command-nodefault gcc : windres :
- [ feature.get-values <rc> : $(options) ] : $(bin) : search-path ] ;
- local rc-type = [ feature.get-values <rc-type> : $(options) ] ;
- rc-type ?= windres ;
- if ! $(rc)
- {
- # If we can not find an RC compiler we fallback to a null one that
- # creates empty object files. This allows the same Jamfiles to work
- # across the board. The null RC uses assembler to create the empty
- # objects, so configure that.
- rc = [ common.get-invocation-command gcc : as : : $(bin) : search-path ]
- ;
- rc-type = null ;
- }
- rc.configure $(rc) : $(condition) : <rc-type>$(rc-type) ;
-}
-
-if [ os.name ] = NT
-{
- # This causes single-line command invocation to not go through .bat files,
- # thus avoiding command-line length limitations.
- # TODO: Set JAMSHELL on specific targets instead of globally.
- JAMSHELL = % ;
-}
-
-generators.register-c-compiler gcc.compile.c++.preprocess : CPP : PREPROCESSED_CPP : <toolset>gcc ;
-generators.register-c-compiler gcc.compile.c.preprocess : C : PREPROCESSED_C : <toolset>gcc ;
-generators.register-c-compiler gcc.compile.c++ : CPP : OBJ : <toolset>gcc ;
-generators.register-c-compiler gcc.compile.c : C : OBJ : <toolset>gcc ;
-generators.register-c-compiler gcc.compile.asm : ASM : OBJ : <toolset>gcc ;
-generators.register-fortran-compiler gcc.compile.fortran : FORTRAN FORTRAN90 : OBJ : <toolset>gcc ;
-
-# pch support
-
-# The compiler looks for a precompiled header in each directory just before it
-# looks for the include file in that directory. The name searched for is the
-# name specified in the #include directive with ".gch" suffix appended. The
-# logic in gcc-pch-generator will make sure that the BASE_PCH suffix is appended
-# to the full header name.
-
-type.set-generated-target-suffix PCH : <toolset>gcc : gch ;
-
-# GCC-specific pch generator.
-class gcc-pch-generator : pch-generator
-{
- import project ;
- import property-set ;
- import type ;
-
- rule run-pch ( project name ? : property-set : sources + )
- {
- # Find the header in sources. Ignore any CPP sources.
- local header ;
- for local s in $(sources)
- {
- if [ type.is-derived [ $(s).type ] H ]
- {
- header = $(s) ;
- }
- }
-
- # Error handling: base header file name should be the same as the base
- # precompiled header name.
- local header-name = [ $(header).name ] ;
- local header-basename = $(header-name:B) ;
- if $(header-basename) != $(name)
- {
- local location = [ $(project).project-module ] ;
- import errors : user-error : errors.user-error ;
- errors.user-error "in" $(location): pch target name '$(name)' should
- be the same as the base name of header file '$(header-name)' ;
- }
-
- local pch-file = [ generator.run $(project) $(name) : $(property-set)
- : $(header) ] ;
-
- # Return result of base class and pch-file property as
- # usage-requirements.
- return
- [ property-set.create <pch-file>$(pch-file) <cflags>-Winvalid-pch ]
- $(pch-file)
- ;
- }
-
- # Calls the base version specifying source's name as the name of the created
- # target. As a result, the PCH will be named whatever.hpp.gch, and not
- # whatever.gch.
- rule generated-targets ( sources + : property-set : project name ? )
- {
- name = [ $(sources[1]).name ] ;
- return [ generator.generated-targets $(sources)
- : $(property-set) : $(project) $(name) ] ;
- }
-}
-
-# Note: the 'H' source type will catch both '.h' header and '.hpp' header. The
-# latter have HPP type, but HPP type is derived from H. The type of compilation
-# is determined entirely by the destination type.
-generators.register [ new gcc-pch-generator gcc.compile.c.pch : H : C_PCH : <pch>on <toolset>gcc ] ;
-generators.register [ new gcc-pch-generator gcc.compile.c++.pch : H : CPP_PCH : <pch>on <toolset>gcc ] ;
-
-# Override default do-nothing generators.
-generators.override gcc.compile.c.pch : pch.default-c-pch-generator ;
-generators.override gcc.compile.c++.pch : pch.default-cpp-pch-generator ;
-
-toolset.flags gcc.compile PCH_FILE <pch>on : <pch-file> ;
-
-# Declare flags and action for compilation.
-toolset.flags gcc.compile OPTIONS <optimization>off : -O0 ;
-toolset.flags gcc.compile OPTIONS <optimization>speed : -O3 ;
-toolset.flags gcc.compile OPTIONS <optimization>space : -Os ;
-
-toolset.flags gcc.compile OPTIONS <inlining>off : -fno-inline ;
-toolset.flags gcc.compile OPTIONS <inlining>on : -Wno-inline ;
-toolset.flags gcc.compile OPTIONS <inlining>full : -finline-functions -Wno-inline ;
-
-toolset.flags gcc.compile OPTIONS <warnings>off : -w ;
-toolset.flags gcc.compile OPTIONS <warnings>on : -Wall ;
-toolset.flags gcc.compile OPTIONS <warnings>all : -Wall -pedantic ;
-toolset.flags gcc.compile OPTIONS <warnings-as-errors>on : -Werror ;
-
-toolset.flags gcc.compile OPTIONS <debug-symbols>on : -g ;
-toolset.flags gcc.compile OPTIONS <profiling>on : -pg ;
-
-toolset.flags gcc.compile.c++ OPTIONS <rtti>off : -fno-rtti ;
-toolset.flags gcc.compile.c++ OPTIONS <exception-handling>off : -fno-exceptions ;
-
-rule setup-fpic ( targets * : sources * : properties * )
-{
- local link = [ feature.get-values link : $(properties) ] ;
- if $(link) = shared
- {
- local target = [ feature.get-values target-os : $(properties) ] ;
-
- # This logic will add -fPIC for all compilations:
- #
- # lib a : a.cpp b ;
- # obj b : b.cpp ;
- # exe c : c.cpp a d ;
- # obj d : d.cpp ;
- #
- # This all is fine, except that 'd' will be compiled with -fPIC even
- # though it is not needed, as 'd' is used only in exe. However, it is
- # hard to detect where a target is going to be used. Alternatively, we
- # can set -fPIC only when main target type is LIB but than 'b' would be
- # compiled without -fPIC which would lead to link errors on x86-64. So,
- # compile everything with -fPIC.
- #
- # Yet another alternative would be to create a propagated <sharedable>
- # feature and set it when building shared libraries, but that would be
- # hard to implement and would increase the target path length even more.
-
- # On Windows, fPIC is the default, and specifying -fPIC explicitly leads
- # to a warning.
- if ! $(target) in cygwin windows
- {
- OPTIONS on $(targets) += -fPIC ;
- }
- }
-}
-
-rule setup-address-model ( targets * : sources * : properties * )
-{
- local model = [ feature.get-values address-model : $(properties) ] ;
- if $(model)
- {
- local option ;
- local os = [ feature.get-values target-os : $(properties) ] ;
- if $(os) = aix
- {
- if $(model) = 32
- {
- option = -maix32 ;
- }
- else
- {
- option = -maix64 ;
- }
- }
- else if $(os) = hpux
- {
- if $(model) = 32
- {
- option = -milp32 ;
- }
- else
- {
- option = -mlp64 ;
- }
- }
- else
- {
- local arch = [ feature.get-values architecture : $(properties) ] ;
- if $(arch) != arm
- {
- if $(model) = 32
- {
- option = -m32 ;
- }
- else if $(model) = 64
- {
- option = -m64 ;
- }
- }
- # For darwin, the model can be 32_64. darwin.jam will handle that
- # on its own.
- }
- OPTIONS on $(targets) += $(option) ;
- }
-}
-
-
-# FIXME: this should not use os.name.
-if ! [ os.name ] in NT OSF HPUX AIX
-{
- # OSF does have an option called -soname but it does not seem to work as
- # expected, therefore it has been disabled.
- HAVE_SONAME = "" ;
- SONAME_OPTION = -h ;
-}
-
-# HPUX, for some reason, seems to use '+h' instead of '-h'.
-if [ os.name ] = HPUX
-{
- HAVE_SONAME = "" ;
- SONAME_OPTION = +h ;
-}
-
-toolset.flags gcc.compile USER_OPTIONS <cflags> ;
-toolset.flags gcc.compile.c++ USER_OPTIONS <cxxflags> ;
-toolset.flags gcc.compile DEFINES <define> ;
-toolset.flags gcc.compile INCLUDES <include> ;
-toolset.flags gcc.compile.c++ TEMPLATE_DEPTH <c++-template-depth> ;
-toolset.flags gcc.compile.fortran USER_OPTIONS <fflags> ;
-
-rule compile.c++.pch ( targets * : sources * : properties * )
-{
- setup-threading $(targets) : $(sources) : $(properties) ;
- setup-fpic $(targets) : $(sources) : $(properties) ;
- setup-address-model $(targets) : $(sources) : $(properties) ;
-}
-
-actions compile.c++.pch
-{
- "$(CONFIG_COMMAND)" -x c++-header $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
-}
-
-rule compile.c.pch ( targets * : sources * : properties * )
-{
- setup-threading $(targets) : $(sources) : $(properties) ;
- setup-fpic $(targets) : $(sources) : $(properties) ;
- setup-address-model $(targets) : $(sources) : $(properties) ;
-}
-
-actions compile.c.pch
-{
- "$(CONFIG_COMMAND)" -x c-header $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
-}
-
-rule compile.c++.preprocess ( targets * : sources * : properties * )
-{
- setup-threading $(targets) : $(sources) : $(properties) ;
- setup-fpic $(targets) : $(sources) : $(properties) ;
- setup-address-model $(targets) : $(sources) : $(properties) ;
-
- # Some extensions are compiled as C++ by default. For others, we need to
- # pass -x c++. We could always pass -x c++ but distcc does not work with it.
- if ! $(>:S) in .cc .cp .cxx .cpp .c++ .C
- {
- LANG on $(<) = "-x c++" ;
- }
- DEPENDS $(<) : [ on $(<) return $(PCH_FILE) ] ;
-}
-
-rule compile.c.preprocess ( targets * : sources * : properties * )
-{
- setup-threading $(targets) : $(sources) : $(properties) ;
- setup-fpic $(targets) : $(sources) : $(properties) ;
- setup-address-model $(targets) : $(sources) : $(properties) ;
-
- # If we use the name g++ then default file suffix -> language mapping does
- # not work. So have to pass -x option. Maybe, we can work around this by
- # allowing the user to specify both C and C++ compiler names.
- #if $(>:S) != .c
- #{
- LANG on $(<) = "-x c" ;
- #}
- DEPENDS $(<) : [ on $(<) return $(PCH_FILE) ] ;
-}
-
-rule compile.c++ ( targets * : sources * : properties * )
-{
- setup-threading $(targets) : $(sources) : $(properties) ;
- setup-fpic $(targets) : $(sources) : $(properties) ;
- setup-address-model $(targets) : $(sources) : $(properties) ;
-
- # Some extensions are compiled as C++ by default. For others, we need to
- # pass -x c++. We could always pass -x c++ but distcc does not work with it.
- if ! $(>:S) in .cc .cp .cxx .cpp .c++ .C
- {
- LANG on $(<) = "-x c++" ;
- }
- DEPENDS $(<) : [ on $(<) return $(PCH_FILE) ] ;
-
- # Here we want to raise the template-depth parameter value to something
- # higher than the default value of 17. Note that we could do this using the
- # feature.set-default rule but we do not want to set the default value for
- # all toolsets as well.
- #
- # TODO: This 'modified default' has been inherited from some 'older Boost
- # Build implementation' and has most likely been added to make some Boost
- # library parts compile correctly. We should see what exactly prompted this
- # and whether we can get around the problem more locally.
- local template-depth = [ on $(<) return $(TEMPLATE_DEPTH) ] ;
- if ! $(template-depth)
- {
- TEMPLATE_DEPTH on $(<) = 128 ;
- }
-}
-
-rule compile.c ( targets * : sources * : properties * )
-{
- setup-threading $(targets) : $(sources) : $(properties) ;
- setup-fpic $(targets) : $(sources) : $(properties) ;
- setup-address-model $(targets) : $(sources) : $(properties) ;
-
- # If we use the name g++ then default file suffix -> language mapping does
- # not work. So have to pass -x option. Maybe, we can work around this by
- # allowing the user to specify both C and C++ compiler names.
- #if $(>:S) != .c
- #{
- LANG on $(<) = "-x c" ;
- #}
- DEPENDS $(<) : [ on $(<) return $(PCH_FILE) ] ;
-}
-
-rule compile.fortran ( targets * : sources * : properties * )
-{
- setup-threading $(targets) : $(sources) : $(properties) ;
- setup-fpic $(targets) : $(sources) : $(properties) ;
- setup-address-model $(targets) : $(sources) : $(properties) ;
-}
-
-actions compile.c++ bind PCH_FILE
-{
- "$(CONFIG_COMMAND)" $(LANG) -ftemplate-depth-$(TEMPLATE_DEPTH) $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(PCH_FILE:D)" -I"$(INCLUDES)" -c -o "$(<:W)" "$(>:W)"
-}
-
-actions compile.c bind PCH_FILE
-{
- "$(CONFIG_COMMAND)" $(LANG) $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(PCH_FILE:D)" -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
-}
-
-actions compile.c++.preprocess bind PCH_FILE
-{
- "$(CONFIG_COMMAND)" $(LANG) -ftemplate-depth-$(TEMPLATE_DEPTH) $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(PCH_FILE:D)" -I"$(INCLUDES)" "$(>:W)" -E >"$(<:W)"
-}
-
-actions compile.c.preprocess bind PCH_FILE
-{
- "$(CONFIG_COMMAND)" $(LANG) $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(PCH_FILE:D)" -I"$(INCLUDES)" "$(>)" -E >$(<)
-}
-
-actions compile.fortran
-{
- "$(CONFIG_COMMAND)" $(LANG) $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(PCH_FILE:D)" -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
-}
-
-rule compile.asm ( targets * : sources * : properties * )
-{
- setup-fpic $(targets) : $(sources) : $(properties) ;
- setup-address-model $(targets) : $(sources) : $(properties) ;
- LANG on $(<) = "-x assembler-with-cpp" ;
-}
-
-actions compile.asm
-{
- "$(CONFIG_COMMAND)" $(LANG) $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
-}
-
-# Class checking that we do not try to use the <runtime-link>static property
-# while creating or using a shared library, since it is not supported by
-# gcc/libc.
-class gcc-linking-generator : unix-linking-generator
-{
- rule run ( project name ? : property-set : sources + )
- {
- # TODO: Replace this with the use of a target-os property.
- local no-static-link = ;
- if [ modules.peek : UNIX ]
- {
- switch [ modules.peek : JAMUNAME ]
- {
- case * : no-static-link = true ;
- }
- }
-
- local properties = [ $(property-set).raw ] ;
- local reason ;
- if $(no-static-link) && <runtime-link>static in $(properties)
- {
- if <link>shared in $(properties)
- {
- reason = On gcc, DLLs can not be built with
- '<runtime-link>static'. ;
- }
- else if [ type.is-derived $(self.target-types[1]) EXE ]
- {
- for local s in $(sources)
- {
- local type = [ $(s).type ] ;
- if $(type) && [ type.is-derived $(type) SHARED_LIB ]
- {
- reason = On gcc, using DLLs together with the
- '<runtime-link>static' option is not possible. ;
- }
- }
- }
- }
- if $(reason)
- {
- ECHO warning: $(reason) ;
- ECHO warning: It is suggested to use '<runtime-link>static' together
- with '<link>static'. ;
- }
- else
- {
- return [ unix-linking-generator.run $(project) $(name) :
- $(property-set) : $(sources) ] ;
- }
- }
-}
-
-# The set of permissible input types is different on mingw. So, define two sets
-# of generators, with mingw generators selected when target-os=windows.
-
-local g ;
-g = [ new gcc-linking-generator gcc.mingw.link
- : OBJ SEARCHED_LIB STATIC_LIB IMPORT_LIB
- : EXE
- : <toolset>gcc <target-os>windows ] ;
-$(g).set-rule-name gcc.link ;
-generators.register $(g) ;
-
-g = [ new gcc-linking-generator gcc.mingw.link.dll
- : OBJ SEARCHED_LIB STATIC_LIB IMPORT_LIB
- : IMPORT_LIB SHARED_LIB
- : <toolset>gcc <target-os>windows ] ;
-$(g).set-rule-name gcc.link.dll ;
-generators.register $(g) ;
-
-generators.register
- [ new gcc-linking-generator gcc.link
- : LIB OBJ
- : EXE
- : <toolset>gcc ] ;
-generators.register
- [ new gcc-linking-generator gcc.link.dll
- : LIB OBJ
- : SHARED_LIB
- : <toolset>gcc ] ;
-
-generators.override gcc.mingw.link : gcc.link ;
-generators.override gcc.mingw.link.dll : gcc.link.dll ;
-
-# Cygwin is similar to msvc and mingw in that it uses import libraries. While in
-# simple cases, it can directly link to a shared library, it is believed to be
-# slower, and not always possible. Define cygwin-specific generators here.
-
-g = [ new gcc-linking-generator gcc.cygwin.link
- : OBJ SEARCHED_LIB STATIC_LIB IMPORT_LIB
- : EXE
- : <toolset>gcc <target-os>cygwin ] ;
-$(g).set-rule-name gcc.link ;
-generators.register $(g) ;
-
-g = [ new gcc-linking-generator gcc.cygwin.link.dll
- : OBJ SEARCHED_LIB STATIC_LIB IMPORT_LIB
- : IMPORT_LIB SHARED_LIB
- : <toolset>gcc <target-os>cygwin ] ;
-$(g).set-rule-name gcc.link.dll ;
-generators.register $(g) ;
-
-generators.override gcc.cygwin.link : gcc.link ;
-generators.override gcc.cygwin.link.dll : gcc.link.dll ;
-
-# Declare flags for linking.
-# First, the common flags.
-toolset.flags gcc.link OPTIONS <debug-symbols>on : -g ;
-toolset.flags gcc.link OPTIONS <profiling>on : -pg ;
-toolset.flags gcc.link USER_OPTIONS <linkflags> ;
-toolset.flags gcc.link LINKPATH <library-path> ;
-toolset.flags gcc.link FINDLIBS-ST <find-static-library> ;
-toolset.flags gcc.link FINDLIBS-SA <find-shared-library> ;
-toolset.flags gcc.link LIBRARIES <library-file> ;
-
-toolset.flags gcc.link.dll .IMPLIB-COMMAND <target-os>windows : "-Wl,--out-implib," ;
-toolset.flags gcc.link.dll .IMPLIB-COMMAND <target-os>cygwin : "-Wl,--out-implib," ;
-
-# For <runtime-link>static we made sure there are no dynamic libraries in the
-# link. On HP-UX not all system libraries exist as archived libraries (for
-# example, there is no libunwind.a), so, on this platform, the -static option
-# cannot be specified.
-if [ os.name ] != HPUX
-{
- toolset.flags gcc.link OPTIONS <runtime-link>static : -static ;
-}
-
-# Now, the vendor specific flags.
-# The parameter linker can be either aix, darwin, gnu, hpux, osf or sun.
-rule init-link-flags ( toolset linker condition )
-{
- switch $(linker)
- {
- case aix :
- # On AIX we *have* to use the native linker.
- #
- # Using -brtl, the AIX linker will look for libraries with both the .a
- # and .so extensions, such as libfoo.a and libfoo.so. Without -brtl, the
- # AIX linker looks only for libfoo.a. Note that libfoo.a is an archived
- # file that may contain shared objects and is different from static libs
- # as on Linux.
- #
- # The -bnoipath strips the prepending (relative) path of libraries from
- # the loader section in the target library or executable. Hence, during
- # load-time LIBPATH (identical to LD_LIBRARY_PATH) or a hard-coded
- # -blibpath (*similar* to -lrpath/-lrpath-link) is searched. Without
- # this option, the prepending (relative) path + library name is
- # hard-coded in the loader section, causing *only* this path to be
- # searched during load-time. Note that the AIX linker does not have an
- # -soname equivalent, this is as close as it gets.
- #
- # The above options are definately for AIX 5.x, and most likely also for
- # AIX 4.x and AIX 6.x. For details about the AIX linker see:
- # http://download.boulder.ibm.com/ibmdl/pub/software/dw/aix/es-aix_ll.pdf
- #
-
- toolset.flags $(toolset).link OPTIONS : -Wl,-brtl -Wl,-bnoipath
- : unchecked ;
-
- case darwin :
- # On Darwin, the -s option to ld does not work unless we pass -static,
- # and passing -static unconditionally is a bad idea. So, do not pass -s
- # at all and darwin.jam will use a separate 'strip' invocation.
- toolset.flags $(toolset).link RPATH $(condition) : <dll-path> :
- unchecked ;
- toolset.flags $(toolset).link RPATH_LINK $(condition) : <xdll-path> :
- unchecked ;
-
- case gnu :
- # Strip the binary when no debugging is needed. We use --strip-all flag
- # as opposed to -s since icc (intel's compiler) is generally
- # option-compatible with and inherits from the gcc toolset, but does not
- # support -s.
- toolset.flags $(toolset).link OPTIONS $(condition)/<strip>on : -Wl,--strip-all : unchecked ;
- toolset.flags $(toolset).link RPATH $(condition) : <dll-path> : unchecked ;
- toolset.flags $(toolset).link RPATH_LINK $(condition) : <xdll-path> : unchecked ;
- toolset.flags $(toolset).link START-GROUP $(condition) : -Wl,--start-group : unchecked ;
- toolset.flags $(toolset).link END-GROUP $(condition) : -Wl,--end-group : unchecked ;
-
- # gnu ld has the ability to change the search behaviour for libraries
- # referenced by the -l switch. These modifiers are -Bstatic and
- # -Bdynamic and change search for -l switches that follow them. The
- # following list shows the tried variants. Search stops at the first
- # variant that has a match.
- #
- # *nix: -Bstatic -lxxx
- # libxxx.a
- #
- # *nix: -Bdynamic -lxxx
- # libxxx.so
- # libxxx.a
- #
- # windows (mingw, cygwin) -Bstatic -lxxx
- # libxxx.a
- # xxx.lib
- #
- # windows (mingw, cygwin) -Bdynamic -lxxx
- # libxxx.dll.a
- # xxx.dll.a
- # libxxx.a
- # xxx.lib
- # cygxxx.dll (*)
- # libxxx.dll
- # xxx.dll
- # libxxx.a
- #
- # (*) This is for cygwin
- # Please note that -Bstatic and -Bdynamic are not a guarantee that a
- # static or dynamic lib indeed gets linked in. The switches only change
- # search patterns!
-
- # On *nix mixing shared libs with static runtime is not a good idea.
- toolset.flags $(toolset).link FINDLIBS-ST-PFX
- $(condition)/<runtime-link>shared : -Wl,-Bstatic : unchecked ;
- toolset.flags $(toolset).link FINDLIBS-SA-PFX
- $(condition)/<runtime-link>shared : -Wl,-Bdynamic : unchecked ;
-
- # On windows allow mixing of static and dynamic libs with static
- # runtime is not a good idea.
- toolset.flags $(toolset).link FINDLIBS-ST-PFX
- $(condition)/<runtime-link>static/<target-os>windows : -Wl,-Bstatic
- : unchecked ;
- toolset.flags $(toolset).link FINDLIBS-SA-PFX
- $(condition)/<runtime-link>static/<target-os>windows : -Wl,-Bdynamic
- : unchecked ;
- toolset.flags $(toolset).link OPTIONS
- $(condition)/<runtime-link>static/<target-os>windows : -Wl,-Bstatic
- : unchecked ;
-
- case hpux :
- toolset.flags $(toolset).link OPTIONS $(condition)/<strip>on : -Wl,-s :
- unchecked ;
- toolset.flags $(toolset).link OPTIONS $(condition)/<link>shared : -fPIC
- : unchecked ;
-
- case osf :
- # No --strip-all, just -s.
- toolset.flags $(toolset).link OPTIONS $(condition)/<strip>on : -Wl,-s :
- unchecked ;
- toolset.flags $(toolset).link RPATH $(condition) : <dll-path> :
- unchecked ;
- # This does not support -R.
- toolset.flags $(toolset).link RPATH_OPTION $(condition) : -rpath :
- unchecked ;
- # -rpath-link is not supported at all.
-
- case sun :
- toolset.flags $(toolset).link OPTIONS $(condition)/<strip>on : -Wl,-s :
- unchecked ;
- toolset.flags $(toolset).link RPATH $(condition) : <dll-path> :
- unchecked ;
- # Solaris linker does not have a separate -rpath-link, but allows using
- # -L for the same purpose.
- toolset.flags $(toolset).link LINKPATH $(condition) : <xdll-path> :
- unchecked ;
-
- # This permits shared libraries with non-PIC code on Solaris.
- # VP, 2004/09/07: Now that we have -fPIC hardcode in link.dll, the
- # following is not needed. Whether -fPIC should be hardcoded, is a
- # separate question.
- # AH, 2004/10/16: it is still necessary because some tests link against
- # static libraries that were compiled without PIC.
- toolset.flags $(toolset).link OPTIONS $(condition)/<link>shared :
- -mimpure-text : unchecked ;
-
- case * :
- import errors ;
- errors.user-error $(toolset) initialization: invalid linker '$(linker)'
- : The value '$(linker)' specified for <linker> is not recognized.
- : Possible values are 'aix', 'darwin', 'gnu', 'hpux', 'osf' or 'sun'
- ;
- }
-}
-
-
-# Enclose the RPATH variable on 'targets' in double quotes, unless it is already
-# enclosed in single quotes. This special casing is done because it is common to
-# pass '$ORIGIN' to linker -- and it has to have single quotes to prevent shell
-# expansion -- and if we add double quotes then the preventing properties of
-# single quotes disappear.
-#
-rule quote-rpath ( targets * )
-{
- local r = [ on $(targets[1]) return $(RPATH) ] ;
- if ! [ MATCH ('.*') : $(r) ]
- {
- r = \"$(r)\" ;
- }
- RPATH on $(targets) = $(r) ;
-}
-
-# Declare actions for linking.
-rule link ( targets * : sources * : properties * )
-{
- setup-threading $(targets) : $(sources) : $(properties) ;
- setup-address-model $(targets) : $(sources) : $(properties) ;
- SPACE on $(targets) = " " ;
- # Serialize execution of the 'link' action, since running N links in
- # parallel is just slower. For now, serialize only gcc links, it might be a
- # good idea to serialize all links.
- JAM_SEMAPHORE on $(targets) = <s>gcc-link-semaphore ;
- quote-rpath $(targets) ;
-}
-
-actions link bind LIBRARIES
-{
- "$(CONFIG_COMMAND)" -L"$(LINKPATH)" -Wl,$(RPATH_OPTION:E=-R)$(SPACE)-Wl,$(RPATH) -Wl,-rpath-link$(SPACE)-Wl,"$(RPATH_LINK)" -o "$(<)" $(START-GROUP) "$(>)" "$(LIBRARIES)" $(FINDLIBS-ST-PFX) -l$(FINDLIBS-ST) $(FINDLIBS-SA-PFX) -l$(FINDLIBS-SA) $(END-GROUP) $(OPTIONS) $(USER_OPTIONS)
-}
-
-
-# Default value. Mostly for the sake of intel-linux that inherits from gcc, but
-# does not have the same logic to set the .AR variable. We can put the same
-# logic in intel-linux, but that is hardly worth the trouble as on Linux, 'ar'
-# is always available.
-.AR = ar ;
-.RANLIB = ranlib ;
-
-toolset.flags gcc.archive AROPTIONS <archiveflags> ;
-
-rule archive ( targets * : sources * : properties * )
-{
- # Always remove archive and start again. Here is the rationale from
- #
- # Andre Hentz:
- #
- # I had a file, say a1.c, that was included into liba.a. I moved a1.c to
- # a2.c, updated my Jamfiles and rebuilt. My program was crashing with absurd
- # errors. After some debugging I traced it back to the fact that a1.o was
- # *still* in liba.a
- #
- # Rene Rivera:
- #
- # Originally removing the archive was done by splicing an RM onto the
- # archive action. That makes archives fail to build on NT when they have
- # many files because it will no longer execute the action directly and blow
- # the line length limit. Instead we remove the file in a different action,
- # just before building the archive.
- #
- local clean.a = $(targets[1])(clean) ;
- TEMPORARY $(clean.a) ;
- NOCARE $(clean.a) ;
- LOCATE on $(clean.a) = [ on $(targets[1]) return $(LOCATE) ] ;
- DEPENDS $(clean.a) : $(sources) ;
- DEPENDS $(targets) : $(clean.a) ;
- common.RmTemps $(clean.a) : $(targets) ;
-}
-
-# Declare action for creating static libraries.
-# The letter 'r' means to add files to the archive with replacement. Since we
-# remove archive, we do not care about replacement, but there is no option "add
-# without replacement".
-# The letter 'c' suppresses the warning in case the archive does not exists yet.
-# That warning is produced only on some platforms, for whatever reasons.
-#
-actions piecemeal archive
-{
- "$(.AR)" $(AROPTIONS) rc "$(<)" "$(>)"
- "$(.RANLIB)" "$(<)"
-}
-
-rule link.dll ( targets * : sources * : properties * )
-{
- setup-threading $(targets) : $(sources) : $(properties) ;
- setup-address-model $(targets) : $(sources) : $(properties) ;
- SPACE on $(targets) = " " ;
- JAM_SEMAPHORE on $(targets) = <s>gcc-link-semaphore ;
- quote-rpath $(targets) ;
-}
-
-# Differs from 'link' above only by -shared.
-actions link.dll bind LIBRARIES
-{
- "$(CONFIG_COMMAND)" -L"$(LINKPATH)" -Wl,$(RPATH_OPTION:E=-R)$(SPACE)-Wl,$(RPATH) "$(.IMPLIB-COMMAND)$(<[1])" -o "$(<[-1])" $(HAVE_SONAME)-Wl,$(SONAME_OPTION)$(SPACE)-Wl,$(<[-1]:D=) -shared $(START-GROUP) "$(>)" "$(LIBRARIES)" $(FINDLIBS-ST-PFX) -l$(FINDLIBS-ST) $(FINDLIBS-SA-PFX) -l$(FINDLIBS-SA) $(END-GROUP) $(OPTIONS) $(USER_OPTIONS)
-}
-
-rule setup-threading ( targets * : sources * : properties * )
-{
- local threading = [ feature.get-values threading : $(properties) ] ;
- if $(threading) = multi
- {
- local target = [ feature.get-values target-os : $(properties) ] ;
- local option ;
- local libs ;
-
- switch $(target)
- {
- case windows : option = -mthreads ;
- case cygwin : option = -mthreads ;
- case solaris : option = -pthreads ; libs = rt ;
- case beos : # No threading options.
- case *bsd : option = -pthread ; # There is no -lrt on BSD.
- case sgi : # gcc on IRIX does not support multi-threading.
- case darwin : # No threading options.
- case * : option = -pthread ; libs = rt ;
- }
-
- if $(option)
- {
- OPTIONS on $(targets) += $(option) ;
- }
- if $(libs)
- {
- FINDLIBS-SA on $(targets) += $(libs) ;
- }
- }
-}
-
-
-local rule cpu-flags ( toolset variable : architecture : instruction-set + :
- values + : default ? )
-{
- if $(default)
- {
- toolset.flags $(toolset) $(variable)
- <architecture>$(architecture)/<instruction-set> : $(values) ;
- }
- toolset.flags $(toolset) $(variable)
- <architecture>/<instruction-set>$(instruction-set)
- <architecture>$(architecture)/<instruction-set>$(instruction-set)
- : $(values) ;
-}
-
-
-# Set architecture/instruction-set options.
-#
-# x86 and compatible
-# The 'native' option appeared in gcc 4.2 so we cannot safely use it as default.
-# Use i686 instead for 32-bit.
-toolset.flags gcc OPTIONS <architecture>x86/<address-model>32/<instruction-set> : -march=i686 ;
-cpu-flags gcc OPTIONS : x86 : native : -march=native ;
-cpu-flags gcc OPTIONS : x86 : i486 : -march=i486 ;
-cpu-flags gcc OPTIONS : x86 : i586 : -march=i586 ;
-cpu-flags gcc OPTIONS : x86 : i686 : -march=i686 ;
-cpu-flags gcc OPTIONS : x86 : pentium : -march=pentium ;
-cpu-flags gcc OPTIONS : x86 : pentium-mmx : -march=pentium-mmx ;
-cpu-flags gcc OPTIONS : x86 : pentiumpro : -march=pentiumpro ;
-cpu-flags gcc OPTIONS : x86 : pentium2 : -march=pentium2 ;
-cpu-flags gcc OPTIONS : x86 : pentium3 : -march=pentium3 ;
-cpu-flags gcc OPTIONS : x86 : pentium3m : -march=pentium3m ;
-cpu-flags gcc OPTIONS : x86 : pentium-m : -march=pentium-m ;
-cpu-flags gcc OPTIONS : x86 : pentium4 : -march=pentium4 ;
-cpu-flags gcc OPTIONS : x86 : pentium4m : -march=pentium4m ;
-cpu-flags gcc OPTIONS : x86 : prescott : -march=prescott ;
-cpu-flags gcc OPTIONS : x86 : nocona : -march=nocona ;
-cpu-flags gcc OPTIONS : x86 : core2 : -march=core2 ;
-cpu-flags gcc OPTIONS : x86 : conroe : -march=core2 ;
-cpu-flags gcc OPTIONS : x86 : conroe-xe : -march=core2 ;
-cpu-flags gcc OPTIONS : x86 : conroe-l : -march=core2 ;
-cpu-flags gcc OPTIONS : x86 : allendale : -march=core2 ;
-cpu-flags gcc OPTIONS : x86 : wolfdale : -march=core2 -msse4.1 ;
-cpu-flags gcc OPTIONS : x86 : merom : -march=core2 ;
-cpu-flags gcc OPTIONS : x86 : merom-xe : -march=core2 ;
-cpu-flags gcc OPTIONS : x86 : kentsfield : -march=core2 ;
-cpu-flags gcc OPTIONS : x86 : kentsfield-xe : -march=core2 ;
-cpu-flags gcc OPTIONS : x86 : yorksfield : -march=core2 ;
-cpu-flags gcc OPTIONS : x86 : penryn : -march=core2 ;
-cpu-flags gcc OPTIONS : x86 : corei7 : -march=corei7 ;
-cpu-flags gcc OPTIONS : x86 : nehalem : -march=corei7 ;
-cpu-flags gcc OPTIONS : x86 : corei7-avx : -march=corei7-avx ;
-cpu-flags gcc OPTIONS : x86 : sandy-bridge : -march=corei7-avx ;
-cpu-flags gcc OPTIONS : x86 : core-avx-i : -march=core-avx-i ;
-cpu-flags gcc OPTIONS : x86 : ivy-bridge : -march=core-avx-i ;
-cpu-flags gcc OPTIONS : x86 : haswell : -march=core-avx-i -mavx2 -mfma -mbmi -mbmi2 -mlzcnt ;
-cpu-flags gcc OPTIONS : x86 : k6 : -march=k6 ;
-cpu-flags gcc OPTIONS : x86 : k6-2 : -march=k6-2 ;
-cpu-flags gcc OPTIONS : x86 : k6-3 : -march=k6-3 ;
-cpu-flags gcc OPTIONS : x86 : athlon : -march=athlon ;
-cpu-flags gcc OPTIONS : x86 : athlon-tbird : -march=athlon-tbird ;
-cpu-flags gcc OPTIONS : x86 : athlon-4 : -march=athlon-4 ;
-cpu-flags gcc OPTIONS : x86 : athlon-xp : -march=athlon-xp ;
-cpu-flags gcc OPTIONS : x86 : athlon-mp : -march=athlon-mp ;
-##
-cpu-flags gcc OPTIONS : x86 : k8 : -march=k8 ;
-cpu-flags gcc OPTIONS : x86 : opteron : -march=opteron ;
-cpu-flags gcc OPTIONS : x86 : athlon64 : -march=athlon64 ;
-cpu-flags gcc OPTIONS : x86 : athlon-fx : -march=athlon-fx ;
-cpu-flags gcc OPTIONS : x86 : k8-sse3 : -march=k8-sse3 ;
-cpu-flags gcc OPTIONS : x86 : opteron-sse3 : -march=opteron-sse3 ;
-cpu-flags gcc OPTIONS : x86 : athlon64-sse3 : -march=athlon64-sse3 ;
-cpu-flags gcc OPTIONS : x86 : amdfam10 : -march=amdfam10 ;
-cpu-flags gcc OPTIONS : x86 : barcelona : -march=barcelona ;
-cpu-flags gcc OPTIONS : x86 : bdver1 : -march=bdver1 ;
-cpu-flags gcc OPTIONS : x86 : bdver2 : -march=bdver2 ;
-cpu-flags gcc OPTIONS : x86 : bdver3 : -march=bdver3 ;
-cpu-flags gcc OPTIONS : x86 : btver1 : -march=btver1 ;
-cpu-flags gcc OPTIONS : x86 : btver2 : -march=btver2 ;
-cpu-flags gcc OPTIONS : x86 : winchip-c6 : -march=winchip-c6 ;
-cpu-flags gcc OPTIONS : x86 : winchip2 : -march=winchip2 ;
-cpu-flags gcc OPTIONS : x86 : c3 : -march=c3 ;
-cpu-flags gcc OPTIONS : x86 : c3-2 : -march=c3-2 ;
-##
-cpu-flags gcc OPTIONS : x86 : atom : -march=atom ;
-# Sparc
-cpu-flags gcc OPTIONS : sparc : c3 : -mcpu=c3 : default ;
-cpu-flags gcc OPTIONS : sparc : v7 : -mcpu=v7 ;
-cpu-flags gcc OPTIONS : sparc : cypress : -mcpu=cypress ;
-cpu-flags gcc OPTIONS : sparc : v8 : -mcpu=v8 ;
-cpu-flags gcc OPTIONS : sparc : supersparc : -mcpu=supersparc ;
-cpu-flags gcc OPTIONS : sparc : sparclite : -mcpu=sparclite ;
-cpu-flags gcc OPTIONS : sparc : hypersparc : -mcpu=hypersparc ;
-cpu-flags gcc OPTIONS : sparc : sparclite86x : -mcpu=sparclite86x ;
-cpu-flags gcc OPTIONS : sparc : f930 : -mcpu=f930 ;
-cpu-flags gcc OPTIONS : sparc : f934 : -mcpu=f934 ;
-cpu-flags gcc OPTIONS : sparc : sparclet : -mcpu=sparclet ;
-cpu-flags gcc OPTIONS : sparc : tsc701 : -mcpu=tsc701 ;
-cpu-flags gcc OPTIONS : sparc : v9 : -mcpu=v9 ;
-cpu-flags gcc OPTIONS : sparc : ultrasparc : -mcpu=ultrasparc ;
-cpu-flags gcc OPTIONS : sparc : ultrasparc3 : -mcpu=ultrasparc3 ;
-# RS/6000 & PowerPC
-cpu-flags gcc OPTIONS : power : 403 : -mcpu=403 ;
-cpu-flags gcc OPTIONS : power : 505 : -mcpu=505 ;
-cpu-flags gcc OPTIONS : power : 601 : -mcpu=601 ;
-cpu-flags gcc OPTIONS : power : 602 : -mcpu=602 ;
-cpu-flags gcc OPTIONS : power : 603 : -mcpu=603 ;
-cpu-flags gcc OPTIONS : power : 603e : -mcpu=603e ;
-cpu-flags gcc OPTIONS : power : 604 : -mcpu=604 ;
-cpu-flags gcc OPTIONS : power : 604e : -mcpu=604e ;
-cpu-flags gcc OPTIONS : power : 620 : -mcpu=620 ;
-cpu-flags gcc OPTIONS : power : 630 : -mcpu=630 ;
-cpu-flags gcc OPTIONS : power : 740 : -mcpu=740 ;
-cpu-flags gcc OPTIONS : power : 7400 : -mcpu=7400 ;
-cpu-flags gcc OPTIONS : power : 7450 : -mcpu=7450 ;
-cpu-flags gcc OPTIONS : power : 750 : -mcpu=750 ;
-cpu-flags gcc OPTIONS : power : 801 : -mcpu=801 ;
-cpu-flags gcc OPTIONS : power : 821 : -mcpu=821 ;
-cpu-flags gcc OPTIONS : power : 823 : -mcpu=823 ;
-cpu-flags gcc OPTIONS : power : 860 : -mcpu=860 ;
-cpu-flags gcc OPTIONS : power : 970 : -mcpu=970 ;
-cpu-flags gcc OPTIONS : power : 8540 : -mcpu=8540 ;
-cpu-flags gcc OPTIONS : power : power : -mcpu=power ;
-cpu-flags gcc OPTIONS : power : power2 : -mcpu=power2 ;
-cpu-flags gcc OPTIONS : power : power3 : -mcpu=power3 ;
-cpu-flags gcc OPTIONS : power : power4 : -mcpu=power4 ;
-cpu-flags gcc OPTIONS : power : power5 : -mcpu=power5 ;
-cpu-flags gcc OPTIONS : power : powerpc : -mcpu=powerpc ;
-cpu-flags gcc OPTIONS : power : powerpc64 : -mcpu=powerpc64 ;
-cpu-flags gcc OPTIONS : power : rios : -mcpu=rios ;
-cpu-flags gcc OPTIONS : power : rios1 : -mcpu=rios1 ;
-cpu-flags gcc OPTIONS : power : rios2 : -mcpu=rios2 ;
-cpu-flags gcc OPTIONS : power : rsc : -mcpu=rsc ;
-cpu-flags gcc OPTIONS : power : rs64a : -mcpu=rs64 ;
-# AIX variant of RS/6000 & PowerPC
-toolset.flags gcc AROPTIONS <address-model>64/<target-os>aix : "-X 64" ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/generate.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/generate.jam b/ext/kenlm/jam-files/boost-build/tools/generate.jam
deleted file mode 100644
index 6732fa3..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/generate.jam
+++ /dev/null
@@ -1,108 +0,0 @@
-# Copyright 2006 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-# Declares main target 'generate' used to produce targets by calling a
-# user-provided rule that takes and produces virtual targets.
-
-import "class" : new ;
-import errors ;
-import feature ;
-import project ;
-import property ;
-import property-set ;
-import targets ;
-import regex ;
-
-
-feature.feature generating-rule : : free ;
-
-
-class generated-target-class : basic-target
-{
- import errors ;
- import indirect ;
- import virtual-target ;
-
- rule __init__ ( name : project : sources * : requirements *
- : default-build * : usage-requirements * )
- {
- basic-target.__init__ $(name) : $(project) : $(sources)
- : $(requirements) : $(default-build) : $(usage-requirements) ;
-
- if ! [ $(self.requirements).get <generating-rule> ]
- {
- errors.user-error "The generate rule requires the <generating-rule>"
- "property to be set" ;
- }
- }
-
- rule construct ( name : sources * : property-set )
- {
- local result ;
- local gr = [ $(property-set).get <generating-rule> ] ;
-
- # FIXME: this is a copy-paste from virtual-target.jam. We should add a
- # utility rule to call a rule like this.
- local rule-name = [ MATCH ^@(.*) : $(gr) ] ;
- if $(rule-name)
- {
- if $(gr[2])
- {
- local target-name = [ full-name ] ;
- errors.user-error "Multiple <generating-rule> properties"
- "encountered for target $(target-name)." ;
- }
-
- result = [ indirect.call $(rule-name) $(self.project) $(name)
- : $(property-set) : $(sources) ] ;
-
- if ! $(result)
- {
- ECHO "warning: Unable to construct" [ full-name ] ;
- }
- }
-
- local ur ;
- local targets ;
-
- if $(result)
- {
- if [ class.is-a $(result[1]) : property-set ]
- {
- ur = $(result[1]) ;
- targets = $(result[2-]) ;
- }
- else
- {
- ur = [ property-set.empty ] ;
- targets = $(result) ;
- }
- }
- # FIXME: the following loop should be doable using sequence.transform or
- # some similar utility rule.
- local rt ;
- for local t in $(targets)
- {
- rt += [ virtual-target.register $(t) ] ;
- }
- return $(ur) $(rt) ;
- }
-}
-
-
-rule generate ( name : sources * : requirements * : default-build *
- : usage-requirements * )
-{
- local project = [ project.current ] ;
-
- targets.main-target-alternative
- [ new generated-target-class $(name) : $(project)
- : [ targets.main-target-sources $(sources) : $(name) ]
- : [ targets.main-target-requirements $(requirements) : $(project) ]
- : [ targets.main-target-default-build $(default-build) : $(project) ]
- : [ targets.main-target-usage-requirements $(usage-requirements) : $(project) ]
- ] ;
-}
-
-IMPORT $(__name__) : generate : : generate ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/gettext.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/gettext.jam b/ext/kenlm/jam-files/boost-build/tools/gettext.jam
deleted file mode 100644
index 99a43ff..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/gettext.jam
+++ /dev/null
@@ -1,230 +0,0 @@
-# Copyright 2003, 2004, 2005, 2006 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-# This module support GNU gettext internationalization utilities.
-#
-# It provides two main target rules: 'gettext.catalog', used for
-# creating machine-readable catalogs from translations files, and
-# 'gettext.update', used for update translation files from modified
-# sources.
-#
-# To add i18n support to your application you should follow these
-# steps.
-#
-# - Decide on a file name which will contain translations and
-# what main target name will be used to update it. For example::
-#
-# gettext.update update-russian : russian.po a.cpp my_app ;
-#
-# - Create the initial translation file by running::
-#
-# bjam update-russian
-#
-# - Edit russian.po. For example, you might change fields like LastTranslator.
-#
-# - Create a main target for final message catalog::
-#
-# gettext.catalog russian : russian.po ;
-#
-# The machine-readable catalog will be updated whenever you update
-# "russian.po". The "russian.po" file will be updated only on explicit
-# request. When you're ready to update translations, you should
-#
-# - Run::
-#
-# bjam update-russian
-#
-# - Edit "russian.po" in appropriate editor.
-#
-# The next bjam run will convert "russian.po" into machine-readable form.
-#
-# By default, translations are marked by 'i18n' call. The 'gettext.keyword'
-# feature can be used to alter this.
-
-
-import targets ;
-import property-set ;
-import virtual-target ;
-import "class" : new ;
-import project ;
-import type ;
-import generators ;
-import errors ;
-import feature : feature ;
-import toolset : flags ;
-import regex ;
-
-.path = "" ;
-
-# Initializes the gettext module.
-rule init ( path ? # Path where all tools are located. If not specified,
- # they should be in PATH.
- )
-{
- if $(.initialized) && $(.path) != $(path)
- {
- errors.error "Attempt to reconfigure with different path" ;
- }
- .initialized = true ;
- if $(path)
- {
- .path = $(path)/ ;
- }
-}
-
-# Creates a main target 'name', which, when updated, will cause
-# file 'existing-translation' to be updated with translations
-# extracted from 'sources'. It's possible to specify main target
-# in sources --- it which case all target from dependency graph
-# of those main targets will be scanned, provided they are of
-# appropricate type. The 'gettext.types' feature can be used to
-# control the types.
-#
-# The target will be updated only if explicitly requested on the
-# command line.
-rule update ( name : existing-translation sources + : requirements * )
-{
- local project = [ project.current ] ;
-
- targets.main-target-alternative
- [ new typed-target $(name) : $(project) : gettext.UPDATE :
- $(existing-translation) $(sources)
- : [ targets.main-target-requirements $(requirements) : $(project) ]
- ] ;
- $(project).mark-target-as-explicit $(name) ;
-}
-
-
-# The human editable source, containing translation.
-type.register gettext.PO : po ;
-# The machine readable message catalog.
-type.register gettext.catalog : mo ;
-# Intermediate type produce by extracting translations from
-# sources.
-type.register gettext.POT : pot ;
-# Pseudo type used to invoke update-translations generator
-type.register gettext.UPDATE ;
-
-# Identifies the keyword that should be used when scanning sources.
-# Default: i18n
-feature gettext.keyword : : free ;
-# Contains space-separated list of sources types which should be scanned.
-# Default: "C CPP"
-feature gettext.types : : free ;
-
-generators.register-standard gettext.compile : gettext.PO : gettext.catalog ;
-
-class update-translations-generator : generator
-{
- import regex : split ;
- import property-set ;
-
- rule __init__ ( * : * )
- {
- generator.__init__ $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ;
- }
-
- # The rule should be called with at least two sources. The first source
- # is the translation (.po) file to update. The remaining sources are targets
- # which should be scanned for new messages. All sources files for those targets
- # will be found and passed to the 'xgettext' utility, which extracts the
- # messages for localization. Those messages will be merged to the .po file.
- rule run ( project name ? : property-set : sources * : multiple ? )
- {
- local types = [ $(property-set).get <gettext.types> ] ;
- types ?= "C CPP" ;
- types = [ regex.split $(types) " " ] ;
-
- local keywords = [ $(property-set).get <gettext.keyword> ] ;
- property-set = [ property-set.create $(keywords:G=<gettext.keyword>) ] ;
-
- # First deterime the list of sources that must be scanned for
- # messages.
- local all-sources ;
- # CONSIDER: I'm not sure if the logic should be the same as for 'stage':
- # i.e. following dependency properties as well.
- for local s in $(sources[2-])
- {
- all-sources += [ virtual-target.traverse $(s) : : include-sources ] ;
- }
- local right-sources ;
- for local s in $(all-sources)
- {
- if [ $(s).type ] in $(types)
- {
- right-sources += $(s) ;
- }
- }
-
- local .constructed ;
- if $(right-sources)
- {
- # Create the POT file, which will contain list of messages extracted
- # from the sources.
- local extract =
- [ new action $(right-sources) : gettext.extract : $(property-set) ] ;
- local new-messages = [ new file-target $(name) : gettext.POT
- : $(project) : $(extract) ] ;
-
- # Create a notfile target which will update the existing translation file
- # with new messages.
- local a = [ new action $(sources[1]) $(new-messages)
- : gettext.update-po-dispatch ] ;
- local r = [ new notfile-target $(name) : $(project) : $(a) ] ;
- .constructed = [ virtual-target.register $(r) ] ;
- }
- else
- {
- errors.error "No source could be scanned by gettext tools" ;
- }
- return $(.constructed) ;
- }
-}
-generators.register [ new update-translations-generator gettext.update : : gettext.UPDATE ] ;
-
-flags gettext.extract KEYWORD <gettext.keyword> ;
-actions extract
-{
- $(.path)xgettext -k$(KEYWORD:E=i18n) -o $(<) $(>)
-}
-
-# Does realy updating of po file. The tricky part is that
-# we're actually updating one of the sources:
-# $(<) is the NOTFILE target we're updating
-# $(>[1]) is the PO file to be really updated.
-# $(>[2]) is the PO file created from sources.
-#
-# When file to be updated does not exist (during the
-# first run), we need to copy the file created from sources.
-# In all other cases, we need to update the file.
-rule update-po-dispatch
-{
- NOCARE $(>[1]) ;
- gettext.create-po $(<) : $(>) ;
- gettext.update-po $(<) : $(>) ;
- _ on $(<) = " " ;
- ok on $(<) = "" ;
- EXISTING_PO on $(<) = $(>[1]) ;
-}
-
-# Due to fancy interaction of existing and updated, this rule can be called with
-# one source, in which case we copy the lonely source into EXISTING_PO, or with
-# two sources, in which case the action body expands to nothing. I'd really like
-# to have "missing" action modifier.
-actions quietly existing updated create-po bind EXISTING_PO
-{
- cp$(_)"$(>[1])"$(_)"$(EXISTING_PO)"$($(>[2]:E=ok))
-}
-
-actions updated update-po bind EXISTING_PO
-{
- $(.path)msgmerge$(_)-U$(_)"$(EXISTING_PO)"$(_)"$(>[1])"
-}
-
-actions gettext.compile
-{
- $(.path)msgfmt -o $(<) $(>)
-}
-
-IMPORT $(__name__) : update : : gettext.update ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/gfortran.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/gfortran.jam b/ext/kenlm/jam-files/boost-build/tools/gfortran.jam
deleted file mode 100644
index 0aa69b8..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/gfortran.jam
+++ /dev/null
@@ -1,39 +0,0 @@
-# Copyright (C) 2004 Toon Knapen
-#
-# Use, modification and distribution is subject to the Boost Software
-# License Version 1.0. (See accompanying file LICENSE_1_0.txt or
-# http://www.boost.org/LICENSE_1_0.txt)
-
-import toolset : flags ;
-import feature ;
-import fortran ;
-
-rule init ( version ? : command * : options * )
-{
-}
-
-# Declare flags and action for compilation
-flags gfortran OPTIONS <fflags> ;
-
-flags gfortran OPTIONS <optimization>off : -O0 ;
-flags gfortran OPTIONS <optimization>speed : -O3 ;
-flags gfortran OPTIONS <optimization>space : -Os ;
-
-flags gfortran OPTIONS <debug-symbols>on : -g ;
-flags gfortran OPTIONS <profiling>on : -pg ;
-
-flags gfortran OPTIONS <link>shared/<main-target-type>LIB : -fPIC ;
-
-flags gfortran DEFINES <define> ;
-flags gfortran INCLUDES <include> ;
-
-rule compile.fortran
-{
-}
-
-actions compile.fortran
-{
- gcc -Wall $(OPTIONS) -D$(DEFINES) -I$(INCLUDES) -c -o "$(<)" "$(>)"
-}
-
-generators.register-fortran-compiler gfortran.compile.fortran : FORTRAN FORTRAN90 : OBJ ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/hp_cxx.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/hp_cxx.jam b/ext/kenlm/jam-files/boost-build/tools/hp_cxx.jam
deleted file mode 100644
index 86cd783..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/hp_cxx.jam
+++ /dev/null
@@ -1,181 +0,0 @@
-# Copyright 2001 David Abrahams.
-# Copyright 2004, 2005 Markus Schoepflin.
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-#
-# HP CXX compiler
-# See http://h30097.www3.hp.com/cplus/?jumpid=reg_R1002_USEN
-#
-#
-# Notes on this toolset:
-#
-# - Because of very subtle issues with the default ansi mode, strict_ansi mode
-# is used for compilation. One example of things that don't work correctly in
-# the default ansi mode is overload resolution of function templates when
-# mixed with non-template functions.
-#
-# - For template instantiation "-timplicit_local" is used. Previously,
-# "-tlocal" has been tried to avoid the need for a template repository
-# but this doesn't work with manually instantiated templates. "-tweak"
-# has not been used to avoid the stream of warning messages issued by
-# ar or ld when creating a library or linking an application.
-#
-# - Debug symbols are generated with "-g3", as this works both in debug and
-# release mode. When compiling C++ code without optimization, we additionally
-# use "-gall", which generates full symbol table information for all classes,
-# structs, and unions. As this turns off optimization, it can't be used when
-# optimization is needed.
-#
-
-import feature generators common ;
-import toolset : flags ;
-
-feature.extend toolset : hp_cxx ;
-feature.extend c++abi : cxxarm ;
-
-# Inherit from Unix toolset to get library ordering magic.
-toolset.inherit hp_cxx : unix ;
-
-generators.override hp_cxx.prebuilt : builtin.lib-generator ;
-generators.override hp_cxx.prebuilt : builtin.prebuilt ;
-generators.override hp_cxx.searched-lib-generator : searched-lib-generator ;
-
-
-rule init ( version ? : command * : options * )
-{
- local condition = [ common.check-init-parameters hp_cxx : version $(version) ] ;
-
- local command = [ common.get-invocation-command hp_cxx : cxx : $(command) ] ;
-
- if $(command)
- {
- local root = [ common.get-absolute-tool-path $(command[-1]) ] ;
-
- if $(root)
- {
- flags hp_cxx .root $(condition) : "\"$(root)\"/" ;
- }
- }
- # If we can't find 'cxx' anyway, at least show 'cxx' in the commands
- command ?= cxx ;
-
- common.handle-options hp_cxx : $(condition) : $(command) : $(options) ;
-}
-
-generators.register-c-compiler hp_cxx.compile.c++ : CPP : OBJ : <toolset>hp_cxx ;
-generators.register-c-compiler hp_cxx.compile.c : C : OBJ : <toolset>hp_cxx ;
-
-
-
-# No static linking as far as I can tell.
-# flags cxx LINKFLAGS <runtime-link>static : -bstatic ;
-flags hp_cxx.compile OPTIONS <debug-symbols>on : -g3 ;
-flags hp_cxx.compile OPTIONS <optimization>off/<debug-symbols>on : -gall ;
-flags hp_cxx.link OPTIONS <debug-symbols>on : -g ;
-flags hp_cxx.link OPTIONS <debug-symbols>off : -s ;
-
-flags hp_cxx.compile OPTIONS <optimization>off : -O0 ;
-flags hp_cxx.compile OPTIONS <optimization>speed/<inlining>on : -O2 ;
-flags hp_cxx.compile OPTIONS <optimization>speed : -O2 ;
-
-# This (undocumented) macro needs to be defined to get all C function
-# overloads required by the C++ standard.
-flags hp_cxx.compile.c++ OPTIONS : -D__CNAME_OVERLOADS ;
-
-# Added for threading support
-flags hp_cxx.compile OPTIONS <threading>multi : -pthread ;
-flags hp_cxx.link OPTIONS <threading>multi : -pthread ;
-
-flags hp_cxx.compile OPTIONS <optimization>space/<inlining>on : <inlining>size ;
-flags hp_cxx.compile OPTIONS <optimization>space : -O1 ;
-flags hp_cxx.compile OPTIONS <inlining>off : -inline none ;
-
-# The compiler versions tried (up to V6.5-040) hang when compiling Boost code
-# with full inlining enabled. So leave it at the default level for now.
-#
-# flags hp_cxx.compile OPTIONS <inlining>full : -inline all ;
-
-flags hp_cxx.compile OPTIONS <profiling>on : -pg ;
-flags hp_cxx.link OPTIONS <profiling>on : -pg ;
-
-# Selection of the object model. This flag is needed on both the C++ compiler
-# and linker command line.
-
-# Unspecified ABI translates to '-model ansi' as most
-# standard-conforming.
-flags hp_cxx.compile.c++ OPTIONS <c++abi> : -model ansi : : hack-hack ;
-flags hp_cxx.compile.c++ OPTIONS <c++abi>cxxarm : -model arm ;
-flags hp_cxx.link OPTIONS <c++abi> : -model ansi : : hack-hack ;
-flags hp_cxx.link OPTIONS <c++abi>cxxarm : -model arm ;
-
-# Display a descriptive tag together with each compiler message. This tag can
-# be used by the user to explicitely suppress the compiler message.
-flags hp_cxx.compile OPTIONS : -msg_display_tag ;
-
-flags hp_cxx.compile OPTIONS <cflags> ;
-flags hp_cxx.compile.c++ OPTIONS <cxxflags> ;
-flags hp_cxx.compile DEFINES <define> ;
-flags hp_cxx.compile INCLUDES <include> ;
-flags hp_cxx.link OPTIONS <linkflags> ;
-
-flags hp_cxx.link LIBPATH <library-path> ;
-flags hp_cxx.link LIBRARIES <library-file> ;
-flags hp_cxx.link FINDLIBS-ST <find-static-library> ;
-flags hp_cxx.link FINDLIBS-SA <find-shared-library> ;
-
-flags hp_cxx.compile.c++ TEMPLATE_DEPTH <c++-template-depth> ;
-
-actions link bind LIBRARIES
-{
- $(CONFIG_COMMAND) -noimplicit_include $(OPTIONS) -o "$(<)" -L$(LIBPATH) "$(>)" "$(LIBRARIES)" -l$(FINDLIBS-ST) -l$(FINDLIBS-SA) -lrt -lm
-}
-
-# When creating dynamic libraries, we don't want to be warned about unresolved
-# symbols, therefore all unresolved symbols are marked as expected by
-# '-expect_unresolved *'. This also mirrors the behaviour of the GNU tool
-# chain.
-
-actions link.dll bind LIBRARIES
-{
- $(CONFIG_COMMAND) -shared -expect_unresolved \* -noimplicit_include $(OPTIONS) -o "$(<[1])" -L$(LIBPATH) "$(>)" "$(LIBRARIES)" -l$(FINDLIBS-ST) -l$(FINDLIBS-SA) -lm
-}
-
-
-# Note: Relaxed ANSI mode (-std) is used for compilation because in strict ANSI
-# C89 mode (-std1) the compiler doesn't accept C++ comments in C files. As -std
-# is the default, no special flag is needed.
-actions compile.c
-{
- $(.root:E=)cc -c $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -o "$(<)" "$(>)"
-}
-
-# Note: The compiler is forced to compile the files as C++ (-x cxx) because
-# otherwise it will silently ignore files with no file extension.
-#
-# Note: We deliberately don't suppress any warnings on the compiler command
-# line, the user can always do this in a customized toolset later on.
-
-rule compile.c++
-{
- # We preprocess the TEMPLATE_DEPTH command line option here because we found
- # no way to do it correctly in the actual action code. There we either get
- # the -pending_instantiations parameter when no c++-template-depth property
- # has been specified or we get additional quotes around
- # "-pending_instantiations ".
- local template-depth = [ on $(1) return $(TEMPLATE_DEPTH) ] ;
- TEMPLATE_DEPTH on $(1) = "-pending_instantiations "$(template-depth) ;
-}
-
-actions compile.c++
-{
- $(CONFIG_COMMAND) -x cxx -c -std strict_ansi -nopure_cname -noimplicit_include -timplicit_local -ptr "$(<[1]:D)/cxx_repository" $(OPTIONS) $(TEMPLATE_DEPTH) -D$(DEFINES) -I"$(INCLUDES)" -o "$(<)" "$(>)"
-}
-
-# Always create archive from scratch. See the gcc toolet for rationale.
-RM = [ common.rm-command ] ;
-actions together piecemeal archive
-{
- $(RM) "$(<)"
- ar rc $(<) $(>)
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/hpfortran.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/hpfortran.jam b/ext/kenlm/jam-files/boost-build/tools/hpfortran.jam
deleted file mode 100644
index 96e8d18..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/hpfortran.jam
+++ /dev/null
@@ -1,35 +0,0 @@
-# Copyright (C) 2004 Toon Knapen
-#
-# Use, modification and distribution is subject to the Boost Software
-# License Version 1.0. (See accompanying file LICENSE_1_0.txt or
-# http://www.boost.org/LICENSE_1_0.txt)
-
-import toolset : flags ;
-import feature ;
-import fortran ;
-
-rule init ( version ? : command * : options * )
-{
-}
-
-# Declare flags and action for compilation
-flags hpfortran OPTIONS <optimization>off : -O0 ;
-flags hpfortran OPTIONS <optimization>speed : -O3 ;
-flags hpfortran OPTIONS <optimization>space : -O1 ;
-
-flags hpfortran OPTIONS <debug-symbols>on : -g ;
-flags hpfortran OPTIONS <profiling>on : -pg ;
-
-flags hpfortran DEFINES <define> ;
-flags hpfortran INCLUDES <include> ;
-
-rule compile.fortran
-{
-}
-
-actions compile.fortran
-{
- f77 +DD64 $(OPTIONS) -D$(DEFINES) -I$(INCLUDES) -c -o "$(<)" "$(>)"
-}
-
-generators.register-fortran-compiler hpfortran.compile.fortran : FORTRAN : OBJ ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/ifort.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/ifort.jam b/ext/kenlm/jam-files/boost-build/tools/ifort.jam
deleted file mode 100644
index eb7c198..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/ifort.jam
+++ /dev/null
@@ -1,44 +0,0 @@
-# Copyright (C) 2004 Toon Knapen
-#
-# Use, modification and distribution is subject to the Boost Software
-# License Version 1.0. (See accompanying file LICENSE_1_0.txt or
-# http://www.boost.org/LICENSE_1_0.txt)
-
-import toolset : flags ;
-import feature ;
-import fortran ;
-
-rule init ( version ? : command * : options * )
-{
-}
-
-# Declare flags and action for compilation
-flags ifort OPTIONS <fflags> ;
-
-flags ifort OPTIONS <optimization>off : /Od ;
-flags ifort OPTIONS <optimization>speed : /O3 ;
-flags ifort OPTIONS <optimization>space : /O1 ;
-
-flags ifort OPTIONS <debug-symbols>on : /debug:full ;
-flags ifort OPTIONS <profiling>on : /Qprof_gen ;
-
-flags ifort.compile FFLAGS <runtime-debugging>off/<runtime-link>shared : /MD ;
-flags ifort.compile FFLAGS <runtime-debugging>on/<runtime-link>shared : /MDd ;
-flags ifort.compile FFLAGS <runtime-debugging>off/<runtime-link>static/<threading>single : /ML ;
-flags ifort.compile FFLAGS <runtime-debugging>on/<runtime-link>static/<threading>single : /MLd ;
-flags ifort.compile FFLAGS <runtime-debugging>off/<runtime-link>static/<threading>multi : /MT ;
-flags ifort.compile FFLAGS <runtime-debugging>on/<runtime-link>static/<threading>multi : /MTd ;
-
-flags ifort DEFINES <define> ;
-flags ifort INCLUDES <include> ;
-
-rule compile.fortran
-{
-}
-
-actions compile.fortran
-{
- ifort $(FFLAGS) $(OPTIONS) /names:lowercase /D$(DEFINES) /I"$(INCLUDES)" /c /object:"$(<)" "$(>)"
-}
-
-generators.register-fortran-compiler ifort.compile.fortran : FORTRAN : OBJ ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/intel-darwin.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/intel-darwin.jam b/ext/kenlm/jam-files/boost-build/tools/intel-darwin.jam
deleted file mode 100644
index c682a7f..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/intel-darwin.jam
+++ /dev/null
@@ -1,227 +0,0 @@
-# Copyright Vladimir Prus 2004.
-# Copyright Noel Belcourt 2007.
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt
-# or copy at http://www.boost.org/LICENSE_1_0.txt)
-
-import intel ;
-import feature : feature ;
-import os ;
-import toolset ;
-import toolset : flags ;
-import gcc ;
-import common ;
-import errors ;
-import generators ;
-
-feature.extend-subfeature toolset intel : platform : darwin ;
-
-toolset.inherit-generators intel-darwin
- <toolset>intel <toolset-intel:platform>darwin
- : gcc
- # Don't inherit PCH generators. They were not tested, and probably
- # don't work for this compiler.
- : gcc.mingw.link gcc.mingw.link.dll gcc.compile.c.pch gcc.compile.c++.pch
- ;
-
-generators.override intel-darwin.prebuilt : builtin.lib-generator ;
-generators.override intel-darwin.prebuilt : builtin.prebuilt ;
-generators.override intel-darwin.searched-lib-generator : searched-lib-generator ;
-
-toolset.inherit-rules intel-darwin : gcc ;
-toolset.inherit-flags intel-darwin : gcc
- : <inlining>off <inlining>on <inlining>full <optimization>space
- <warnings>off <warnings>all <warnings>on
- <architecture>x86/<address-model>32
- <architecture>x86/<address-model>64
- ;
-
-if [ MATCH (--debug-configuration) : [ modules.peek : ARGV ] ]
-{
- .debug-configuration = true ;
-}
-
-# vectorization diagnostics
-feature vectorize : off on full ;
-
-# Initializes the intel-darwin toolset
-# version in mandatory
-# name (default icc) is used to invoke the specified intel complier
-# compile and link options allow you to specify addition command line options for each version
-rule init ( version ? : command * : options * )
-{
- local condition = [ common.check-init-parameters intel-darwin
- : version $(version) ] ;
-
- command = [ common.get-invocation-command intel-darwin : icc
- : $(command) : /opt/intel_cc_80/bin ] ;
-
- common.handle-options intel-darwin : $(condition) : $(command) : $(options) ;
-
- gcc.init-link-flags intel-darwin darwin $(condition) ;
-
- # handle <library-path>
- # local library-path = [ feature.get-values <library-path> : $(options) ] ;
- # flags intel-darwin.link USER_OPTIONS $(condition) : [ feature.get-values <dll-path> : $(options) ] ;
-
- local root = [ feature.get-values <root> : $(options) ] ;
- local bin ;
- if $(command) || $(root)
- {
- bin ?= [ common.get-absolute-tool-path $(command[-1]) ] ;
- root ?= $(bin:D) ;
-
- if $(root)
- {
- # Libraries required to run the executable may be in either
- # $(root)/lib (10.1 and earlier)
- # or
- # $(root)/lib/architecture-name (11.0 and later:
- local lib_path = $(root)/lib $(root:P)/lib/$(bin:B) ;
- if $(.debug-configuration)
- {
- ECHO notice: using intel libraries :: $(condition) :: $(lib_path) ;
- }
- flags intel-darwin.link RUN_PATH $(condition) : $(lib_path) ;
- }
- }
-
- local m = [ MATCH (..).* : $(version) ] ;
- local n = [ MATCH (.)\\. : $(m) ] ;
- if $(n) {
- m = $(n) ;
- }
-
- local major = $(m) ;
-
- if $(major) = "9" {
- flags intel-darwin.compile OPTIONS $(condition)/<inlining>off : -Ob0 ;
- flags intel-darwin.compile OPTIONS $(condition)/<inlining>on : -Ob1 ;
- flags intel-darwin.compile OPTIONS $(condition)/<inlining>full : -Ob2 ;
- flags intel-darwin.compile OPTIONS $(condition)/<vectorize>off : -vec-report0 ;
- flags intel-darwin.compile OPTIONS $(condition)/<vectorize>on : -vec-report1 ;
- flags intel-darwin.compile OPTIONS $(condition)/<vectorize>full : -vec-report5 ;
- flags intel-darwin.link OPTIONS $(condition)/<runtime-link>static : -static -static-libcxa -lstdc++ -lpthread ;
- flags intel-darwin.link OPTIONS $(condition)/<runtime-link>shared : -shared-libcxa -lstdc++ -lpthread ;
- }
- else {
- flags intel-darwin.compile OPTIONS $(condition)/<inlining>off : -inline-level=0 ;
- flags intel-darwin.compile OPTIONS $(condition)/<inlining>on : -inline-level=1 ;
- flags intel-darwin.compile OPTIONS $(condition)/<inlining>full : -inline-level=2 ;
- flags intel-darwin.compile OPTIONS $(condition)/<vectorize>off : -vec-report0 ;
- flags intel-darwin.compile OPTIONS $(condition)/<vectorize>on : -vec-report1 ;
- flags intel-darwin.compile OPTIONS $(condition)/<vectorize>full : -vec-report5 ;
- flags intel-darwin.link OPTIONS $(condition)/<runtime-link>static : -static -static-intel -lstdc++ -lpthread ;
- flags intel-darwin.link OPTIONS $(condition)/<runtime-link>shared : -shared-intel -lstdc++ -lpthread ;
- }
-
- local minor = [ MATCH ".*\\.(.).*" : $(version) ] ;
-
- # wchar_t char_traits workaround for compilers older than 10.2
- if $(major) = "9" || ( $(major) = "10" && ( $(minor) = "0" || $(minor) = "1" ) ) {
- flags intel-darwin.compile DEFINES $(condition) : __WINT_TYPE__=int : unchecked ;
- }
-}
-
-SPACE = " " ;
-
-flags intel-darwin.compile OPTIONS <cflags> ;
-flags intel-darwin.compile OPTIONS <cxxflags> ;
-# flags intel-darwin.compile INCLUDES <include> ;
-
-flags intel-darwin.compile OPTIONS <optimization>space : -O1 ; # no specific space optimization flag in icc
-
-#
-.cpu-type-em64t = prescott nocona core2 corei7 corei7-avx core-avx-i
- conroe conroe-xe conroe-l allendale merom
- merom-xe kentsfield kentsfield-xe penryn wolfdale
- yorksfield nehalem sandy-bridge ivy-bridge haswell ;
-.cpu-type-amd64 = k8 opteron athlon64 athlon-fx k8-sse3 opteron-sse3
- athlon64-sse3 amdfam10 barcelona bdver1 bdver2 bdver3 btver1 btver2 ;
-.cpu-type-x86-64 = $(.cpu-type-em64t) $(.cpu-type-amd64) ;
-
-flags intel-darwin.compile OPTIONS <instruction-set>$(.cpu-type-x86-64)/<address-model>32 : -m32 ; # -mcmodel=small ;
-flags intel-darwin.compile OPTIONS <instruction-set>$(.cpu-type-x86-64)/<address-model>64 : -m64 ; # -mcmodel=large ;
-
-flags intel-darwin.compile.c OPTIONS <warnings>off : -w0 ;
-flags intel-darwin.compile.c OPTIONS <warnings>on : -w1 ;
-flags intel-darwin.compile.c OPTIONS <warnings>all : -w2 ;
-
-flags intel-darwin.compile.c++ OPTIONS <warnings>off : -w0 ;
-flags intel-darwin.compile.c++ OPTIONS <warnings>on : -w1 ;
-flags intel-darwin.compile.c++ OPTIONS <warnings>all : -w2 ;
-
-actions compile.c
-{
- "$(CONFIG_COMMAND)" -xc $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
-}
-
-actions compile.c++
-{
- "$(CONFIG_COMMAND)" -xc++ $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
-}
-
-flags intel-darwin ARFLAGS <archiveflags> ;
-
-# Default value. Mostly for the sake of intel-linux
-# that inherits from gcc, but does not has the same
-# logic to set the .AR variable. We can put the same
-# logic in intel-linux, but that's hardly worth the trouble
-# as on Linux, 'ar' is always available.
-.AR = ar ;
-
-rule archive ( targets * : sources * : properties * )
-{
- # Always remove archive and start again. Here's rationale from
- # Andre Hentz:
- #
- # I had a file, say a1.c, that was included into liba.a.
- # I moved a1.c to a2.c, updated my Jamfiles and rebuilt.
- # My program was crashing with absurd errors.
- # After some debugging I traced it back to the fact that a1.o was *still*
- # in liba.a
- #
- # Rene Rivera:
- #
- # Originally removing the archive was done by splicing an RM
- # onto the archive action. That makes archives fail to build on NT
- # when they have many files because it will no longer execute the
- # action directly and blow the line length limit. Instead we
- # remove the file in a different action, just before the building
- # of the archive.
- #
- local clean.a = $(targets[1])(clean) ;
- TEMPORARY $(clean.a) ;
- NOCARE $(clean.a) ;
- LOCATE on $(clean.a) = [ on $(targets[1]) return $(LOCATE) ] ;
- DEPENDS $(clean.a) : $(sources) ;
- DEPENDS $(targets) : $(clean.a) ;
- common.RmTemps $(clean.a) : $(targets) ;
-}
-
-actions piecemeal archive
-{
- "$(.AR)" $(AROPTIONS) rc "$(<)" "$(>)"
- "ranlib" -cs "$(<)"
-}
-
-flags intel-darwin.link USER_OPTIONS <linkflags> ;
-
-# Declare actions for linking
-rule link ( targets * : sources * : properties * )
-{
- SPACE on $(targets) = " " ;
- # Serialize execution of the 'link' action, since
- # running N links in parallel is just slower.
- JAM_SEMAPHORE on $(targets) = <s>intel-darwin-link-semaphore ;
-}
-
-actions link bind LIBRARIES
-{
- "$(CONFIG_COMMAND)" $(USER_OPTIONS) -L"$(LINKPATH)" -o "$(<)" "$(>)" "$(LIBRARIES)" -l$(FINDLIBS-SA) -l$(FINDLIBS-ST) $(OPTIONS)
-}
-
-actions link.dll bind LIBRARIES
-{
- "$(CONFIG_COMMAND)" $(USER_OPTIONS) -L"$(LINKPATH)" -o "$(<)" -single_module -dynamiclib -install_name "$(<[1]:D=)" "$(>)" "$(LIBRARIES)" -l$(FINDLIBS-SA) -l$(FINDLIBS-ST) $(OPTIONS)
-}
[36/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/png.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/png.jam b/ext/kenlm/jam-files/boost-build/tools/png.jam
deleted file mode 100644
index 0544fe9..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/png.jam
+++ /dev/null
@@ -1,226 +0,0 @@
-# Copyright (c) 2010 Vladimir Prus.
-# Copyright (c) 2013 Steven Watanabe
-#
-# Use, modification and distribution is subject to the Boost Software
-# License Version 1.0. (See accompanying file LICENSE_1_0.txt or
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# Supports the libpng library
-#
-# After 'using libpng', the following targets are available:
-#
-# /libpng//libpng -- The libpng library
-
-import project ;
-import ac ;
-import errors ;
-import "class" : new ;
-import targets ;
-import path ;
-import modules ;
-import errors ;
-import indirect ;
-import property ;
-import property-set ;
-
-header = png.h ;
-names = libpng ;
-
-sources = png.c pngerror.c pngget.c pngmem.c pngpread.c pngread.c pngrio.c pngrtran.c pngrutil.c
- pngset.c pngtrans.c pngwio.c pngwrite.c pngwtran.c pngwutil.c ;
-
-library-id = 0 ;
-
-if --debug-configuration in [ modules.peek : ARGV ]
-{
- .debug = true ;
-}
-
-# Initializes the libpng library.
-#
-# libpng can be configured either to use pre-existing binaries
-# or to build the library from source.
-#
-# Options for configuring a prebuilt libpng::
-#
-# <search>
-# The directory containing the libpng binaries.
-# <name>
-# Overrides the default library name.
-# <include>
-# The directory containing the libpng headers.
-#
-# If none of these options is specified, then the environmental
-# variables LIBPNG_LIBRARY_PATH, LIBPNG_NAME, and LIBPNG_INCLUDE will
-# be used instead.
-#
-# Options for building libpng from source::
-#
-# <source>
-# The libpng source directory. Defaults to the environmental variable
-# LIBPNG_SOURCE.
-# <tag>
-# A rule which computes the actual name of the compiled
-# libraries based on the build properties. Ignored
-# when using precompiled binaries.
-# <build-name>
-# The base name to use for the compiled library. Ignored
-# when using precompiled binaries.
-#
-# Examples::
-#
-# # Find libpng in the default system location
-# using libpng ;
-# # Build libpng from source
-# using libpng : 1.5.4 : <source>/home/steven/libpng-1.5.4 ;
-# # Find libpng in /usr/local
-# using libpng : 1.5.4
-# : <include>/usr/local/include <search>/usr/local/lib ;
-# # Build libpng from source for msvc and find
-# # prebuilt binaries for gcc.
-# using libpng : 1.5.4 : <source>C:/Devel/src/libpng-1.5.4 : <toolset>msvc ;
-# using libpng : 1.5.4 : : <toolset>gcc ;
-#
-rule init (
- version ?
- # The libpng version (currently ignored)
-
- : options *
- # A list of the options to use
-
- : requirements *
- # The requirements for the libpng target
-
- : is-default ?
- # Default configurations are only used when libpng
- # has not yet been configured.
- )
-{
- local caller = [ project.current ] ;
-
- if ! $(.initialized)
- {
- .initialized = true ;
-
- project.initialize $(__name__) ;
- .project = [ project.current ] ;
- project libpng ;
- }
-
- local library-path = [ property.select <search> : $(options) ] ;
- library-path = $(library-path:G=) ;
- local include-path = [ property.select <include> : $(options) ] ;
- include-path = $(include-path:G=) ;
- local source-path = [ property.select <source> : $(options) ] ;
- source-path = $(source-path:G=) ;
- local library-name = [ property.select <name> : $(options) ] ;
- library-name = $(library-name:G=) ;
- local tag = [ property.select <tag> : $(options) ] ;
- tag = $(tag:G=) ;
- local build-name = [ property.select <build-name> : $(options) ] ;
- build-name = $(build-name:G=) ;
-
- condition = [ property-set.create $(requirements) ] ;
- condition = [ property-set.create [ $(condition).base ] ] ;
-
- local no-build-from-source ;
- # Ignore environmental ZLIB_SOURCE if this initialization
- # requested to search for a specific pre-built library.
- if $(library-path) || $(include-path) || $(library-name)
- {
- if $(source-path) || $(tag) || $(build-name)
- {
- errors.user-error "incompatible options for libpng:"
- [ property.select <search> <include> <name> : $(options) ] "and"
- [ property.select <source> <tag> <build-name> : $(options) ] ;
- }
- else
- {
- no-build-from-source = true ;
- }
- }
-
- source-path ?= [ modules.peek : ZLIB_SOURCE ] ;
-
- if $(.configured.$(condition))
- {
- if $(is-default)
- {
- if $(.debug)
- {
- ECHO "notice: [libpng] libpng is already configured" ;
- }
- }
- else
- {
- errors.user-error "libpng is already configured" ;
- }
- return ;
- }
- else if $(source-path) && ! $(no-build-from-source)
- {
- build-name ?= z ;
- library-id = [ CALC $(library-id) + 1 ] ;
- tag = [ MATCH ^@?(.*)$ : $(tag) ] ;
- if $(tag) && ! [ MATCH ^([^%]*)%([^%]+)$ : $(tag) ]
- {
- tag = [ indirect.make $(tag) : [ $(caller).project-module ] ] ;
- }
- sources = [ path.glob $(source-path) : $(sources) ] ;
- if $(.debug)
- {
- ECHO "notice: [libpng] Building libpng from source as $(build-name)" ;
- if $(condition)
- {
- ECHO "notice: [libpng] Condition" [ $(condition).raw ] ;
- }
- if $(sources)
- {
- ECHO "notice: [libpng] found libpng source in $(source-path)" ;
- }
- else
- {
- ECHO "warning: [libpng] could not find libpng source in $(source-path)" ;
- }
- }
- local target ;
- if $(sources) {
- target = [ targets.create-typed-target LIB : $(.project)
- : $(build-name).$(library-id)
- : $(sources)
- : $(requirements)
- <tag>@$(tag)
- <include>$(source-path)
- <toolset>msvc:<define>_CRT_SECURE_NO_DEPRECATE
- <toolset>msvc:<define>_SCL_SECURE_NO_DEPRECATE
- <link>shared:<define>ZLIB_DLL
- :
- : <include>$(source-path) ] ;
- }
-
- local mt = [ new ac-library libpng : $(.project) : $(condition) ] ;
- $(mt).set-header $(header) ;
- $(mt).set-default-names $(names) ;
- if $(target)
- {
- $(mt).set-target $(target) ;
- }
- targets.main-target-alternative $(mt) ;
- } else {
- if $(.debug)
- {
- ECHO "notice: [libpng] Using pre-installed library" ;
- if $(condition)
- {
- ECHO "notice: [libpng] Condition" [ $(condition).raw ] ;
- }
- }
-
- local mt = [ new ac-library libpng : $(.project) : $(condition) :
- $(include-path) : $(library-path) : $(library-name) : $(root) ] ;
- $(mt).set-header $(header) ;
- $(mt).set-default-names $(names) ;
- targets.main-target-alternative $(mt) ;
- }
- .configured.$(condition) = true ;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/python-config.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/python-config.jam b/ext/kenlm/jam-files/boost-build/tools/python-config.jam
deleted file mode 100644
index 40aa825..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/python-config.jam
+++ /dev/null
@@ -1,27 +0,0 @@
-#~ Copyright 2005 Rene Rivera.
-#~ Distributed under the Boost Software License, Version 1.0.
-#~ (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-# Automatic configuration for Python tools and librries. To use, just import this module.
-
-import os ;
-import toolset : using ;
-
-if [ os.name ] = NT
-{
- for local R in 2.4 2.3 2.2
- {
- local python-path = [ W32_GETREG
- "HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\$(R)\\InstallPath" ] ;
- local python-version = $(R) ;
-
- if $(python-path)
- {
- if --debug-configuration in [ modules.peek : ARGV ]
- {
- ECHO "notice:" using python ":" $(python-version) ":" $(python-path) ;
- }
- using python : $(python-version) : $(python-path) ;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/python.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/python.jam b/ext/kenlm/jam-files/boost-build/tools/python.jam
deleted file mode 100644
index a50a516..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/python.jam
+++ /dev/null
@@ -1,1258 +0,0 @@
-# Copyright 2004 Vladimir Prus.
-# Distributed under the Boost Software License, Version 1.0. (See
-# accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# Support for Python and the the Boost.Python library.
-#
-# This module defines
-#
-# - a project 'python' with a target 'python' in it, that corresponds to the
-# python library
-#
-# - a main target rule 'python-extension' which can be used to build a python
-# extension.
-#
-# Extensions that use Boost.Python must explicitly link to it.
-
-import type ;
-import testing ;
-import generators ;
-import project ;
-import errors ;
-import targets ;
-import "class" : new ;
-import os ;
-import common ;
-import toolset ;
-import regex ;
-import numbers ;
-import string ;
-import property ;
-import sequence ;
-import path ;
-import feature ;
-import set ;
-import builtin ;
-
-
-# Make this module a project.
-project.initialize $(__name__) ;
-project python ;
-
-# Save the project so that if 'init' is called several times we define new
-# targets in the python project, not in whatever project we were called by.
-.project = [ project.current ] ;
-
-# Dynamic linker lib. Necessary to specify it explicitly on some platforms.
-lib dl ;
-# This contains 'openpty' function need by python. Again, on some system need to
-# pass this to linker explicitly.
-lib util ;
-# Python uses pthread symbols.
-lib pthread ;
-# Extra library needed by phtread on some platforms.
-lib rt ;
-
-# The pythonpath feature specifies additional elements for the PYTHONPATH
-# environment variable, set by run-pyd. For example, pythonpath can be used to
-# access Python modules that are part of the product being built, but are not
-# installed in the development system's default paths.
-feature.feature pythonpath : : free optional path ;
-
-# Initializes the Python toolset. Note that all parameters are optional.
-#
-# - version -- the version of Python to use. Should be in Major.Minor format,
-# for example 2.3. Do not include the subminor version.
-#
-# - cmd-or-prefix: Preferably, a command that invokes a Python interpreter.
-# Alternatively, the installation prefix for Python libraries and includes. If
-# empty, will be guessed from the version, the platform's installation
-# patterns, and the python executables that can be found in PATH.
-#
-# - includes: the include path to Python headers. If empty, will be guessed.
-#
-# - libraries: the path to Python library binaries. If empty, will be guessed.
-# On MacOS/Darwin, you can also pass the path of the Python framework.
-#
-# - condition: if specified, should be a set of properties that are matched
-# against the build configuration when Boost.Build selects a Python
-# configuration to use.
-#
-# - extension-suffix: A string to append to the name of extension modules before
-# the true filename extension. Ordinarily we would just compute this based on
-# the value of the <python-debugging> feature. However ubuntu's python-dbg
-# package uses the windows convention of appending _d to debug-build extension
-# modules. We have no way of detecting ubuntu, or of probing python for the
-# "_d" requirement, and if you configure and build python using
-# --with-pydebug, you'll be using the standard *nix convention. Defaults to ""
-# (or "_d" when targeting windows and <python-debugging> is set).
-#
-# Example usage:
-#
-# using python : 2.3 ;
-# using python : 2.3 : /usr/local/bin/python ;
-#
-rule init ( version ? : cmd-or-prefix ? : includes * : libraries ?
- : condition * : extension-suffix ? )
-{
- project.push-current $(.project) ;
-
- debug-message Configuring python... ;
- for local v in version cmd-or-prefix includes libraries condition
- {
- if $($(v))
- {
- debug-message " user-specified "$(v): \"$($(v))\" ;
- }
- }
-
- configure $(version) : $(cmd-or-prefix) : $(includes) : $(libraries) : $(condition) : $(extension-suffix) ;
-
- project.pop-current ;
-}
-
-# A simpler version of SHELL that grabs stderr as well as stdout, but returns
-# nothing if there was an error.
-#
-local rule shell-cmd ( cmd )
-{
- debug-message running command '$(cmd)" 2>&1"' ;
- x = [ SHELL $(cmd)" 2>&1" : exit-status ] ;
- if $(x[2]) = 0
- {
- return $(x[1]) ;
- }
- else
- {
- return ;
- }
-}
-
-
-# Try to identify Cygwin symlinks. Invoking such a file directly as an NT
-# executable from a native Windows build of bjam would be fatal to the bjam
-# process. One /can/ invoke them through sh.exe or bash.exe, if you can prove
-# that those are not also symlinks. ;-)
-#
-# If a symlink is found returns non-empty; we try to extract the target of the
-# symlink from the file and return that.
-#
-# Note: 1. only works on NT 2. path is a native path.
-local rule is-cygwin-symlink ( path )
-{
- local is-symlink = ;
-
- # Look for a file with the given path having the S attribute set, as cygwin
- # symlinks do. /-C means "do not use thousands separators in file sizes."
- local dir-listing = [ shell-cmd "DIR /-C /A:S \""$(path)"\"" ] ;
-
- if $(dir-listing)
- {
- # Escape any special regex characters in the base part of the path.
- local base-pat = [ regex.escape $(path:D=) : ].[()*+?|\\$^ : \\ ] ;
-
- # Extract the file's size from the directory listing.
- local size-of-system-file = [ MATCH "([0-9]+) "$(base-pat) : $(dir-listing) : 1 ] ;
-
- # If the file has a reasonably small size, look for the special symlink
- # identification text.
- if $(size-of-system-file) && [ numbers.less $(size-of-system-file) 1000 ]
- {
- local link = [ SHELL "FIND /OFF \"!<symlink>\" \""$(path)"\" 2>&1" ] ;
- if $(link[2]) != 0
- {
- local nl = "
-
-" ;
- is-symlink = [ MATCH ".*!<symlink>([^"$(nl)"]*)" : $(link[1]) : 1 ] ;
- if $(is-symlink)
- {
- is-symlink = [ *nix-path-to-native $(is-symlink) ] ;
- is-symlink = $(is-symlink:R=$(path:D)) ;
- }
-
- }
- }
- }
- return $(is-symlink) ;
-}
-
-
-# Append ext to each member of names that does not contain '.'.
-#
-local rule default-extension ( names * : ext * )
-{
- local result ;
- for local n in $(names)
- {
- switch $(n)
- {
- case *.* : result += $(n) ;
- case * : result += $(n)$(ext) ;
- }
- }
- return $(result) ;
-}
-
-
-# Tries to determine whether invoking "cmd" would actually attempt to launch a
-# cygwin symlink.
-#
-# Note: only works on NT.
-#
-local rule invokes-cygwin-symlink ( cmd )
-{
- local dirs = $(cmd:D) ;
- if ! $(dirs)
- {
- dirs = . [ os.executable-path ] ;
- }
- local base = [ default-extension $(cmd:D=) : .exe .cmd .bat ] ;
- local paths = [ GLOB $(dirs) : $(base) ] ;
- if $(paths)
- {
- # Make sure we have not run into a Cygwin symlink. Invoking such a file
- # as an NT executable would be fatal for the bjam process.
- return [ is-cygwin-symlink $(paths[1]) ] ;
- }
-}
-
-
-local rule debug-message ( message * )
-{
- if --debug-configuration in [ modules.peek : ARGV ]
- {
- ECHO notice: [python-cfg] $(message) ;
- }
-}
-
-
-# Like W32_GETREG, except prepend HKEY_CURRENT_USER\SOFTWARE and
-# HKEY_LOCAL_MACHINE\SOFTWARE to the first argument, returning the first result
-# found. Also accounts for the fact that on 64-bit machines, 32-bit software has
-# its own area, under SOFTWARE\Wow6432node.
-#
-local rule software-registry-value ( path : data ? )
-{
- local result ;
- for local root in HKEY_CURRENT_USER HKEY_LOCAL_MACHINE
- {
- for local x64elt in "" Wow6432node\\ # Account for 64-bit windows
- {
- if ! $(result)
- {
- result = [ W32_GETREG $(root)\\SOFTWARE\\$(x64elt)$(path) : $(data) ] ;
- }
- }
-
- }
- return $(result) ;
-}
-
-
-.windows-drive-letter-re = ^([A-Za-z]):[\\/](.*) ;
-.cygwin-drive-letter-re = ^/cygdrive/([a-z])/(.*) ;
-
-.working-directory = [ PWD ] ;
-.working-drive-letter = [ SUBST $(.working-directory) $(.windows-drive-letter-re) $1 ] ;
-.working-drive-letter ?= [ SUBST $(.working-directory) $(.cygwin-drive-letter-re) $1 ] ;
-
-
-local rule windows-to-cygwin-path ( path )
-{
- # If path is rooted with a drive letter, rewrite it using the /cygdrive
- # mountpoint.
- local p = [ SUBST $(path:T) $(.windows-drive-letter-re) /cygdrive/$1/$2 ] ;
-
- # Else if path is rooted without a drive letter, use the working directory.
- p ?= [ SUBST $(path:T) ^/(.*) /cygdrive/$(.working-drive-letter:L)/$2 ] ;
-
- # Else return the path unchanged.
- return $(p:E=$(path:T)) ;
-}
-
-
-# :W only works in Cygwin builds of bjam. This one works on NT builds as well.
-#
-local rule cygwin-to-windows-path ( path )
-{
- path = $(path:R="") ; # strip any trailing slash
-
- local drive-letter = [ SUBST $(path) $(.cygwin-drive-letter-re) $1:/$2 ] ;
- if $(drive-letter)
- {
- path = $(drive-letter) ;
- }
- else if $(path:R=/x) = $(path) # already rooted?
- {
- # Look for a cygwin mount that includes each head sequence in $(path).
- local head = $(path) ;
- local tail = "" ;
-
- while $(head)
- {
- local root = [ software-registry-value
- "Cygnus Solutions\\Cygwin\\mounts v2\\"$(head) : native ] ;
-
- if $(root)
- {
- path = $(tail:R=$(root)) ;
- head = ;
- }
- tail = $(tail:R=$(head:D=)) ;
-
- if $(head) = /
- {
- head = ;
- }
- else
- {
- head = $(head:D) ;
- }
- }
- }
- return [ regex.replace $(path:R="") / \\ ] ;
-}
-
-
-# Convert a *nix path to native.
-#
-local rule *nix-path-to-native ( path )
-{
- if [ os.name ] = NT
- {
- path = [ cygwin-to-windows-path $(path) ] ;
- }
- return $(path) ;
-}
-
-
-# Convert an NT path to native.
-#
-local rule windows-path-to-native ( path )
-{
- if [ os.name ] = NT
- {
- return $(path) ;
- }
- else
- {
- return [ windows-to-cygwin-path $(path) ] ;
- }
-}
-
-
-# Return nonempty if path looks like a windows path, i.e. it starts with a drive
-# letter or contains backslashes.
-#
-local rule guess-windows-path ( path )
-{
- return [ SUBST $(path) ($(.windows-drive-letter-re)|.*([\\]).*) $1 ] ;
-}
-
-
-local rule path-to-native ( paths * )
-{
- local result ;
-
- for local p in $(paths)
- {
- if [ guess-windows-path $(p) ]
- {
- result += [ windows-path-to-native $(p) ] ;
- }
- else
- {
- result += [ *nix-path-to-native $(p:T) ] ;
- }
- }
- return $(result) ;
-}
-
-
-# Validate the version string and extract the major/minor part we care about.
-#
-local rule split-version ( version )
-{
- local major-minor = [ MATCH ^([0-9]+)\.([0-9]+)(.*)$ : $(version) : 1 2 3 ] ;
- if ! $(major-minor[2]) || $(major-minor[3])
- {
- ECHO "Warning: \"using python\" expects a two part (major, minor) version number; got" $(version) instead ;
-
- # Add a zero to account for the missing digit if necessary.
- major-minor += 0 ;
- }
-
- return $(major-minor[1]) $(major-minor[2]) ;
-}
-
-
-# Build a list of versions from 3.0 down to 1.5. Because bjam can not enumerate
-# registry sub-keys, we have no way of finding a version with a 2-digit minor
-# version, e.g. 2.10 -- let us hope that never happens.
-#
-.version-countdown = ;
-for local v in [ numbers.range 15 30 ]
-{
- .version-countdown = [ SUBST $(v) (.)(.*) $1.$2 ] $(.version-countdown) ;
-}
-
-
-local rule windows-installed-pythons ( version ? )
-{
- version ?= $(.version-countdown) ;
- local interpreters ;
-
- for local v in $(version)
- {
- local install-path = [
- software-registry-value "Python\\PythonCore\\"$(v)"\\InstallPath" ] ;
-
- if $(install-path)
- {
- install-path = [ windows-path-to-native $(install-path) ] ;
- debug-message Registry indicates Python $(v) installed at \"$(install-path)\" ;
- }
-
- interpreters += $(:E=python:R=$(install-path)) ;
- }
- return $(interpreters) ;
-}
-
-
-local rule darwin-installed-pythons ( version ? )
-{
- version ?= $(.version-countdown) ;
-
- local prefix
- = [ GLOB /System/Library/Frameworks /Library/Frameworks
- : Python.framework ] ;
-
- return $(prefix)/Versions/$(version)/bin/python ;
-}
-
-
-# Assume "python-cmd" invokes a python interpreter and invoke it to extract all
-# the information we care about from its "sys" module. Returns void if
-# unsuccessful.
-#
-local rule probe ( python-cmd )
-{
- # Avoid invoking a Cygwin symlink on NT.
- local skip-symlink ;
- if [ os.name ] = NT
- {
- skip-symlink = [ invokes-cygwin-symlink $(python-cmd) ] ;
- }
-
- if $(skip-symlink)
- {
- debug-message -------------------------------------------------------------------- ;
- debug-message \"$(python-cmd)\" would attempt to invoke a Cygwin symlink, ;
- debug-message causing a bjam built for Windows to hang. ;
- debug-message ;
- debug-message If you intend to target a Cygwin build of Python, please ;
- debug-message replace the path to the link with the path to a real executable ;
- debug-message (guessing: \"$(skip-symlink)\") "in" your 'using python' line ;
- debug-message "in" user-config.jam or site-config.jam. Do not forget to escape ;
- debug-message backslashes ;
- debug-message -------------------------------------------------------------------- ;
- }
- else
- {
- # Prepare a List of Python format strings and expressions that can be
- # used to print the constants we want from the sys module.
-
- # We do not really want sys.version since that is a complicated string,
- # so get the information from sys.version_info instead.
- local format = "version=%d.%d" ;
- local exprs = "version_info[0]" "version_info[1]" ;
-
- for local s in $(sys-elements[2-])
- {
- format += $(s)=%s ;
- exprs += $(s) ;
- }
-
- # Invoke Python and ask it for all those values.
- local full-cmd =
- $(python-cmd)" -c \"from sys import *; print('"$(format:J=\\n)"' % ("$(exprs:J=,)"))\"" ;
-
- local output = [ shell-cmd $(full-cmd) ] ;
- if $(output)
- {
- # Parse the output to get all the results.
- local nl = "
-
-" ;
- for s in $(sys-elements)
- {
- # These variables are expected to be declared local in the
- # caller, so Jam's dynamic scoping will set their values there.
- sys.$(s) = [ SUBST $(output) \\<$(s)=([^$(nl)]+) $1 ] ;
- }
- }
- return $(output) ;
- }
-}
-
-
-# Make sure the "libraries" and "includes" variables (in an enclosing scope)
-# have a value based on the information given.
-#
-local rule compute-default-paths ( target-os : version ? : prefix ? :
- exec-prefix ? )
-{
- exec-prefix ?= $(prefix) ;
-
- if $(target-os) = windows
- {
- # The exec_prefix is where you're supposed to look for machine-specific
- # libraries.
- local default-library-path = $(exec-prefix)\\libs ;
- local default-include-path = $(:E=Include:R=$(prefix)) ;
-
- # If the interpreter was found in a directory called "PCBuild" or
- # "PCBuild8," assume we're looking at a Python built from the source
- # distro, and go up one additional level to the default root. Otherwise,
- # the default root is the directory where the interpreter was found.
-
- # We ask Python itself what the executable path is in case of
- # intermediate symlinks or shell scripts.
- local executable-dir = $(sys.executable:D) ;
-
- if [ MATCH ^(PCBuild) : $(executable-dir:D=) ]
- {
- debug-message "This Python appears to reside in a source distribution;" ;
- debug-message "prepending \""$(executable-dir)"\" to default library search path" ;
-
- default-library-path = $(executable-dir) $(default-library-path) ;
-
- default-include-path = $(:E=PC:R=$(executable-dir:D)) $(default-include-path) ;
-
- debug-message "and \""$(default-include-path[1])"\" to default #include path" ;
- }
-
- libraries ?= $(default-library-path) ;
- includes ?= $(default-include-path) ;
- }
- else
- {
- includes ?= $(prefix)/include/python$(version) ;
-
- local lib = $(exec-prefix)/lib ;
- libraries ?= $(lib)/python$(version)/config $(lib) ;
- }
-}
-
-# The version of the python interpreter to use.
-feature.feature python : : propagated ;
-feature.feature python.interpreter : : free ;
-
-toolset.flags python.capture-output PYTHON : <python.interpreter> ;
-
-#
-# Support for Python configured --with-pydebug
-#
-feature.feature python-debugging : off on : propagated ;
-builtin.variant debug-python : debug : <python-debugging>on ;
-
-
-# Return a list of candidate commands to try when looking for a Python
-# interpreter. prefix is expected to be a native path.
-#
-local rule candidate-interpreters ( version ? : prefix ? : target-os )
-{
- local bin-path = bin ;
- if $(target-os) = windows
- {
- # On Windows, look in the root directory itself and, to work with the
- # result of a build-from-source, the PCBuild directory.
- bin-path = PCBuild8 PCBuild "" ;
- }
-
- bin-path = $(bin-path:R=$(prefix)) ;
-
- if $(target-os) in windows darwin
- {
- return # Search:
- $(:E=python:R=$(bin-path)) # Relative to the prefix, if any
- python # In the PATH
- [ $(target-os)-installed-pythons $(version) ] # Standard install locations
- ;
- }
- else
- {
- # Search relative to the prefix, or if none supplied, in PATH.
- local unversioned = $(:E=python:R=$(bin-path:E=)) ;
-
- # If a version was specified, look for a python with that specific
- # version appended before looking for one called, simply, "python"
- return $(unversioned)$(version) $(unversioned) ;
- }
-}
-
-
-# Compute system library dependencies for targets linking with static Python
-# libraries.
-#
-# On many systems, Python uses libraries such as pthreads or libdl. Since static
-# libraries carry no library dependency information of their own that the linker
-# can extract, these extra dependencies have to be given explicitly on the link
-# line of the client. The information about these dependencies is packaged into
-# the "python" target below.
-#
-# Even where Python itself uses pthreads, it never allows extension modules to
-# be entered concurrently (unless they explicitly give up the interpreter lock).
-# Therefore, extension modules do not need the efficiency overhead of threadsafe
-# code as produced by <threading>multi, and we handle libpthread along with
-# other libraries here. Note: this optimization is based on an assumption that
-# the compiler generates link-compatible code in both the single- and
-# multi-threaded cases, and that system libraries do not change their ABIs
-# either.
-#
-# Returns a list of usage-requirements that link to the necessary system
-# libraries.
-#
-local rule system-library-dependencies ( target-os )
-{
- switch $(target-os)
- {
- case s[uo][nl]* : # solaris, sun, sunos
- # Add a librt dependency for the gcc toolset on SunOS (the sun
- # toolset adds -lrt unconditionally). While this appears to
- # duplicate the logic already in gcc.jam, it does not as long as
- # we are not forcing <threading>multi.
-
- # On solaris 10, distutils.sysconfig.get_config_var('LIBS') yields
- # '-lresolv -lsocket -lnsl -lrt -ldl'. However, that does not seem
- # to be the right list for extension modules. For example, on my
- # installation, adding -ldl causes at least one test to fail because
- # the library can not be found and removing it causes no failures.
-
- # Apparently, though, we need to add -lrt for gcc.
- return <toolset>gcc:<library>rt ;
-
- case osf : return <library>pthread <toolset>gcc:<library>rt ;
-
- case qnx* : return ;
- case darwin : return ;
- case windows : return ;
-
- case hpux : return <library>rt ;
- case *bsd : return <library>pthread <toolset>gcc:<library>util ;
-
- case aix : return <library>pthread <library>dl ;
-
- case * : return <library>pthread <library>dl
- <toolset>gcc:<library>util <toolset-intel:platform>linux:<library>util ;
- }
-}
-
-
-# Declare a target to represent Python's library.
-#
-local rule declare-libpython-target ( version ? : requirements * )
-{
- # Compute the representation of Python version in the name of Python's
- # library file.
- local lib-version = $(version) ;
- if <target-os>windows in $(requirements)
- {
- local major-minor = [ split-version $(version) ] ;
- lib-version = $(major-minor:J="") ;
- if <python-debugging>on in $(requirements)
- {
- lib-version = $(lib-version)_d ;
- }
- }
-
- if ! $(lib-version)
- {
- ECHO *** warning: could not determine Python version, which will ;
- ECHO *** warning: probably prevent us from linking with the python ;
- ECHO *** warning: library. Consider explicitly passing the version ;
- ECHO *** warning: to 'using python'. ;
- }
-
- # Declare it.
- lib python.lib : : <name>python$(lib-version) $(requirements) ;
-}
-
-
-# Implementation of init.
-local rule configure ( version ? : cmd-or-prefix ? : includes * : libraries ? :
- condition * : extension-suffix ? )
-{
- local prefix ;
- local exec-prefix ;
- local cmds-to-try ;
- local interpreter-cmd ;
-
- local target-os = [ feature.get-values target-os : $(condition) ] ;
- target-os ?= [ feature.defaults target-os ] ;
- target-os = $(target-os:G=) ;
-
- if $(target-os) = windows && <python-debugging>on in $(condition)
- {
- extension-suffix ?= _d ;
- }
- extension-suffix ?= "" ;
-
- # Normalize and dissect any version number.
- local major-minor ;
- if $(version)
- {
- major-minor = [ split-version $(version) ] ;
- version = $(major-minor:J=.) ;
- }
-
- local cmds-to-try ;
-
- if ! $(cmd-or-prefix) || [ GLOB $(cmd-or-prefix) : * ]
- {
- # If the user did not pass a command, whatever we got was a prefix.
- prefix = $(cmd-or-prefix) ;
- cmds-to-try = [ candidate-interpreters $(version) : $(prefix) : $(target-os) ] ;
- }
- else
- {
- # Work with the command the user gave us.
- cmds-to-try = $(cmd-or-prefix) ;
-
- # On Windows, do not nail down the interpreter command just yet in case
- # the user specified something that turns out to be a cygwin symlink,
- # which could bring down bjam if we invoke it.
- if $(target-os) != windows
- {
- interpreter-cmd = $(cmd-or-prefix) ;
- }
- }
-
- # Values to use in case we can not really find anything in the system.
- local fallback-cmd = $(cmds-to-try[1]) ;
- local fallback-version ;
-
- # Anything left to find or check?
- if ! ( $(interpreter-cmd) && $(includes) && $(libraries) )
- {
- # Values to be extracted from python's sys module. These will be set by
- # the probe rule, above, using Jam's dynamic scoping.
- local sys-elements = version platform prefix exec_prefix executable ;
- local sys.$(sys-elements) ;
-
- # Compute the string Python's sys.platform needs to match. If not
- # targeting Windows or cygwin we will assume only native builds can
- # possibly run, so we will not require a match and we leave sys.platform
- # blank.
- local platform ;
- switch $(target-os)
- {
- case windows : platform = win32 ;
- case cygwin : platform = cygwin ;
- }
-
- while $(cmds-to-try)
- {
- # Pop top command.
- local cmd = $(cmds-to-try[1]) ;
- cmds-to-try = $(cmds-to-try[2-]) ;
-
- debug-message Checking interpreter command \"$(cmd)\"... ;
- if [ probe $(cmd) ]
- {
- fallback-version ?= $(sys.version) ;
-
- # Check for version/platform validity.
- for local x in version platform
- {
- if $($(x)) && $($(x)) != $(sys.$(x))
- {
- debug-message ...$(x) "mismatch (looking for"
- $($(x)) but found $(sys.$(x))")" ;
- cmd = ;
- }
- }
-
- if $(cmd)
- {
- debug-message ...requested configuration matched! ;
-
- exec-prefix = $(sys.exec_prefix) ;
-
- compute-default-paths $(target-os) : $(sys.version) :
- $(sys.prefix) : $(sys.exec_prefix) ;
-
- version = $(sys.version) ;
- interpreter-cmd ?= $(cmd) ;
- cmds-to-try = ; # All done.
- }
- }
- else
- {
- debug-message ...does not invoke a working interpreter ;
- }
- }
- }
-
- # Anything left to compute?
- if $(includes) && $(libraries)
- {
- .configured = true ;
- }
- else
- {
- version ?= $(fallback-version) ;
- version ?= 2.5 ;
- exec-prefix ?= $(prefix) ;
- compute-default-paths $(target-os) : $(version) : $(prefix:E=) ;
- }
-
- if ! $(interpreter-cmd)
- {
- fallback-cmd ?= python ;
- debug-message No working Python interpreter found. ;
- if [ os.name ] != NT || ! [ invokes-cygwin-symlink $(fallback-cmd) ]
- {
- interpreter-cmd = $(fallback-cmd) ;
- debug-message falling back to \"$(interpreter-cmd)\" ;
- }
- }
-
- includes = [ path-to-native $(includes) ] ;
- libraries = [ path-to-native $(libraries) ] ;
-
- debug-message "Details of this Python configuration:" ;
- debug-message " interpreter command:" \"$(interpreter-cmd:E=<empty>)\" ;
- debug-message " include path:" \"$(includes:E=<empty>)\" ;
- debug-message " library path:" \"$(libraries:E=<empty>)\" ;
- if $(target-os) = windows
- {
- debug-message " DLL search path:" \"$(exec-prefix:E=<empty>)\" ;
- }
-
- #
- # End autoconfiguration sequence.
- #
- local target-requirements = $(condition) ;
-
- # Add the version, if any, to the target requirements.
- if $(version)
- {
- if ! $(version) in [ feature.values python ]
- {
- feature.extend python : $(version) ;
- }
- target-requirements += <python>$(version:E=default) ;
- }
-
- target-requirements += <target-os>$(target-os) ;
-
- # See if we can find a framework directory on darwin.
- local framework-directory ;
- if $(target-os) = darwin
- {
- # Search upward for the framework directory.
- local framework-directory = $(libraries[-1]) ;
- while $(framework-directory:D=) && $(framework-directory:D=) != Python.framework
- {
- framework-directory = $(framework-directory:D) ;
- }
-
- if $(framework-directory:D=) = Python.framework
- {
- debug-message framework directory is \"$(framework-directory)\" ;
- }
- else
- {
- debug-message "no framework directory found; using library path" ;
- framework-directory = ;
- }
- }
-
- local dll-path = $(libraries) ;
-
- # Make sure that we can find the Python DLL on Windows.
- if ( $(target-os) = windows ) && $(exec-prefix)
- {
- dll-path += $(exec-prefix) ;
- }
-
- #
- # Prepare usage requirements.
- #
- local usage-requirements = [ system-library-dependencies $(target-os) ] ;
- usage-requirements += <include>$(includes) <python.interpreter>$(interpreter-cmd) ;
- if <python-debugging>on in $(condition)
- {
- if $(target-os) = windows
- {
- # In pyconfig.h, Py_DEBUG is set if _DEBUG is set. If we define
- # Py_DEBUG we will get multiple definition warnings.
- usage-requirements += <define>_DEBUG ;
- }
- else
- {
- usage-requirements += <define>Py_DEBUG ;
- }
- }
-
- # Global, but conditional, requirements to give access to the interpreter
- # for general utilities, like other toolsets, that run Python scripts.
- toolset.add-requirements
- $(target-requirements:J=,):<python.interpreter>$(interpreter-cmd) ;
-
- # Register the right suffix for extensions.
- register-extension-suffix $(extension-suffix) : $(target-requirements) ;
-
- #
- # Declare the "python" target. This should really be called
- # python_for_embedding.
- #
-
- if $(framework-directory)
- {
- alias python
- :
- : $(target-requirements)
- :
- : $(usage-requirements) <framework>$(framework-directory)
- ;
- }
- else
- {
- declare-libpython-target $(version) : $(target-requirements) ;
-
- # This is an evil hack. On, Windows, when Python is embedded, nothing
- # seems to set up sys.path to include Python's standard library
- # (http://article.gmane.org/gmane.comp.python.general/544986). The evil
- # here, aside from the workaround necessitated by Python's bug, is that:
- #
- # a. we're guessing the location of the python standard library from the
- # location of pythonXX.lib
- #
- # b. we're hijacking the <testing.launcher> property to get the
- # environment variable set up, and the user may want to use it for
- # something else (e.g. launch the debugger).
- local set-PYTHONPATH ;
- if $(target-os) = windows
- {
- set-PYTHONPATH = [ common.prepend-path-variable-command PYTHONPATH :
- $(libraries:D)/Lib ] ;
- }
-
- alias python
- :
- : $(target-requirements)
- :
- # Why python.lib must be listed here instead of along with the
- # system libs is a mystery, but if we do not do it, on cygwin,
- # -lpythonX.Y never appears in the command line (although it does on
- # linux).
- : $(usage-requirements)
- <testing.launcher>$(set-PYTHONPATH)
- <library-path>$(libraries) <library>python.lib
- ;
- }
-
- # On *nix, we do not want to link either Boost.Python or Python extensions
- # to libpython, because the Python interpreter itself provides all those
- # symbols. If we linked to libpython, we would get duplicate symbols. So
- # declare two targets -- one for building extensions and another for
- # embedding.
- #
- # Unlike most *nix systems, Mac OS X's linker does not permit undefined
- # symbols when linking a shared library. So, we still need to link against
- # the Python framework, even when building extensions. Note that framework
- # builds of Python always use shared libraries, so we do not need to worry
- # about duplicate Python symbols.
- if $(target-os) in windows cygwin darwin
- {
- alias python_for_extensions : python : $(target-requirements) ;
- }
- # On AIX we need Python extensions and Boost.Python to import symbols from
- # the Python interpreter. Dynamic libraries opened with dlopen() do not
- # inherit the symbols from the Python interpreter.
- else if $(target-os) = aix
- {
- alias python_for_extensions
- :
- : $(target-requirements)
- :
- : $(usage-requirements) <linkflags>-Wl,-bI:$(libraries[1])/python.exp
- ;
- }
- else
- {
- alias python_for_extensions
- :
- : $(target-requirements)
- :
- : $(usage-requirements)
- ;
- }
-}
-
-
-rule configured ( )
-{
- return $(.configured) ;
-}
-
-
-type.register PYTHON_EXTENSION : : SHARED_LIB ;
-
-
-local rule register-extension-suffix ( root : condition * )
-{
- local suffix ;
-
- switch [ feature.get-values target-os : $(condition) ]
- {
- case windows : suffix = pyd ;
- case cygwin : suffix = dll ;
- case hpux :
- {
- if [ feature.get-values python : $(condition) ] in 1.5 1.6 2.0 2.1 2.2 2.3 2.4
- {
- suffix = sl ;
- }
- else
- {
- suffix = so ;
- }
- }
- case * : suffix = so ;
- }
-
- type.set-generated-target-suffix PYTHON_EXTENSION : $(condition) : <$(root).$(suffix)> ;
-}
-
-
-# Unset 'lib' prefix for PYTHON_EXTENSION
-type.set-generated-target-prefix PYTHON_EXTENSION : : "" ;
-
-
-rule python-extension ( name : sources * : requirements * : default-build * :
- usage-requirements * )
-{
- if [ configured ]
- {
- requirements += <use>/python//python_for_extensions ;
- }
- requirements += <suppress-import-lib>true ;
-
- local project = [ project.current ] ;
-
- targets.main-target-alternative
- [ new typed-target $(name) : $(project) : PYTHON_EXTENSION
- : [ targets.main-target-sources $(sources) : $(name) ]
- : [ targets.main-target-requirements $(requirements) : $(project) ]
- : [ targets.main-target-default-build $(default-build) : $(project) ]
- ] ;
-}
-
-IMPORT python : python-extension : : python-extension ;
-
-rule py2to3
-{
- common.copy $(>) $(<) ;
- 2to3 $(<) ;
-}
-
-actions 2to3
-{
- 2to3 -wn "$(<)"
- 2to3 -dwn "$(<)"
-}
-
-
-# Support for testing.
-type.register PY : py ;
-type.register RUN_PYD_OUTPUT ;
-type.register RUN_PYD : : TEST ;
-
-
-class python-test-generator : generator
-{
- import set ;
-
- rule __init__ ( * : * )
- {
- generator.__init__ $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ;
- self.composing = true ;
- }
-
- rule run ( project name ? : property-set : sources * : multiple ? )
- {
- local pyversion = [ $(property-set).get <python> ] ;
- local python ;
- local other-pythons ;
-
- # Make new target that converting Python source by 2to3 when running with Python 3.
- local rule make-2to3-source ( source )
- {
- if $(pyversion) >= 3.0
- {
- local a = [ new action $(source) : python.py2to3 : $(property-set) ] ;
- local t = [ utility.basename [ $(s).name ] ] ;
- local p = [ new file-target $(t) : PY : $(project) : $(a) ] ;
- return $(p) ;
- }
- else
- {
- return $(source) ;
- }
- }
-
- for local s in $(sources)
- {
- if [ $(s).type ] = PY
- {
- if ! $(python)
- {
- # First Python source ends up on command line.
- python = [ make-2to3-source $(s) ] ;
-
- }
- else
- {
- # Other Python sources become dependencies.
- other-pythons += [ make-2to3-source $(s) ] ;
- }
- }
- }
-
- local extensions ;
- for local s in $(sources)
- {
- if [ $(s).type ] = PYTHON_EXTENSION
- {
- extensions += $(s) ;
- }
- }
-
- local libs ;
- for local s in $(sources)
- {
- if [ type.is-derived [ $(s).type ] LIB ]
- && ! $(s) in $(extensions)
- {
- libs += $(s) ;
- }
- }
-
- local new-sources ;
- for local s in $(sources)
- {
- if [ type.is-derived [ $(s).type ] CPP ]
- {
- local name = [ utility.basename [ $(s).name ] ] ;
- if $(name) = [ utility.basename [ $(python).name ] ]
- {
- name = $(name)_ext ;
- }
- local extension = [ generators.construct $(project) $(name) :
- PYTHON_EXTENSION : $(property-set) : $(s) $(libs) ] ;
-
- # The important part of usage requirements returned from
- # PYTHON_EXTENSION generator are xdll-path properties that will
- # allow us to find the python extension at runtime.
- property-set = [ $(property-set).add $(extension[1]) ] ;
-
- # Ignore usage requirements. We're a top-level generator and
- # nobody is going to use what we generate.
- new-sources += $(extension[2-]) ;
- }
- }
-
- property-set = [ $(property-set).add-raw <dependency>$(other-pythons) ] ;
-
- return [ construct-result $(python) $(extensions) $(new-sources) :
- $(project) $(name) : $(property-set) ] ;
- }
-}
-
-
-generators.register
- [ new python-test-generator python.capture-output : : RUN_PYD_OUTPUT ] ;
-
-generators.register-standard testing.expect-success
- : RUN_PYD_OUTPUT : RUN_PYD ;
-
-
-# There are two different ways of spelling OS names. One is used for [ os.name ]
-# and the other is used for the <host-os> and <target-os> properties. Until that
-# is remedied, this sets up a crude mapping from the latter to the former, that
-# will work *for the purposes of cygwin/NT cross-builds only*. Could not think
-# of a better name than "translate".
-#
-.translate-os-windows = NT ;
-.translate-os-cygwin = CYGWIN ;
-local rule translate-os ( src-os )
-{
- local x = $(.translate-os-$(src-os)) [ os.name ] ;
- return $(x[1]) ;
-}
-
-
-# Extract the path to a single ".pyd" source. This is used to build the
-# PYTHONPATH for running bpl tests.
-#
-local rule pyd-pythonpath ( source )
-{
- return [ on $(source) return $(LOCATE) $(SEARCH) ] ;
-}
-
-
-# The flag settings on testing.capture-output do not apply to python.capture
-# output at the moment. Redo this explicitly.
-toolset.flags python.capture-output ARGS <testing.arg> ;
-
-
-rule capture-output ( target : sources * : properties * )
-{
- # Setup up a proper DLL search path. Here, $(sources[1]) is a python module
- # and $(sources[2]) is a DLL. Only $(sources[1]) is passed to
- # testing.capture-output, so RUN_PATH variable on $(sources[2]) is not
- # consulted. Move it over explicitly.
- RUN_PATH on $(sources[1]) = [ on $(sources[2-]) return $(RUN_PATH) ] ;
-
- PYTHONPATH = [ sequence.transform pyd-pythonpath : $(sources[2-]) ] ;
- PYTHONPATH += [ feature.get-values pythonpath : $(properties) ] ;
-
- # After test is run, we remove the Python module, but not the Python script.
- testing.capture-output $(target) : $(sources[1]) : $(properties) :
- $(sources[2-]) ;
-
- # PYTHONPATH is different; it will be interpreted by whichever Python is
- # invoked and so must follow path rules for the target os. The only OSes
- # where we can run python for other OSes currently are NT and CYGWIN so we
- # only need to handle those cases.
- local target-os = [ feature.get-values target-os : $(properties) ] ;
- # Oddly, host-os is not in properties, so grab the default value.
- local host-os = [ feature.defaults host-os ] ;
- host-os = $(host-os:G=) ;
- if $(target-os) != $(host-os)
- {
- PYTHONPATH = [ sequence.transform $(host-os)-to-$(target-os)-path :
- $(PYTHONPATH) ] ;
- }
- local path-separator = [ os.path-separator [ translate-os $(target-os) ] ] ;
- local set-PYTHONPATH = [ common.variable-setting-command PYTHONPATH :
- $(PYTHONPATH:J=$(path-separator)) ] ;
- LAUNCHER on $(target) = $(set-PYTHONPATH) [ on $(target) return \"$(PYTHON)\" ] ;
-}
-
-
-rule bpl-test ( name : sources * : requirements * )
-{
- local s ;
- sources ?= $(name).py $(name).cpp ;
- return [ testing.make-test run-pyd : $(sources) /boost/python//boost_python
- : $(requirements) : $(name) ] ;
-}
-
-
-IMPORT $(__name__) : bpl-test : : bpl-test ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/qcc.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/qcc.jam b/ext/kenlm/jam-files/boost-build/tools/qcc.jam
deleted file mode 100644
index 3b35578..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/qcc.jam
+++ /dev/null
@@ -1,238 +0,0 @@
-# Copyright (c) 2001 David Abrahams.
-# Copyright (c) 2002-2003 Rene Rivera.
-# Copyright (c) 2002-2003 Vladimir Prus.
-#
-# Use, modification and distribution is subject to the Boost Software
-# License Version 1.0. (See accompanying file LICENSE_1_0.txt or
-# http://www.boost.org/LICENSE_1_0.txt)
-
-import "class" : new ;
-import common ;
-import errors ;
-import feature ;
-import generators ;
-import os ;
-import property ;
-import set ;
-import toolset ;
-import type ;
-import unix ;
-
-feature.extend toolset : qcc ;
-
-toolset.inherit-generators qcc : unix : unix.link unix.link.dll ;
-generators.override builtin.lib-generator : qcc.prebuilt ;
-toolset.inherit-flags qcc : unix ;
-toolset.inherit-rules qcc : unix ;
-
-# Initializes the qcc toolset for the given version. If necessary, command may
-# be used to specify where the compiler is located. The parameter 'options' is a
-# space-delimited list of options, each one being specified as
-# <option-name>option-value. Valid option names are: cxxflags, linkflags and
-# linker-type. Accepted values for linker-type are gnu and sun, gnu being the
-# default.
-#
-# Example:
-# using qcc : 3.4 : : <cxxflags>foo <linkflags>bar <linker-type>sun ;
-#
-rule init ( version ? : command * : options * )
-{
- local condition = [ common.check-init-parameters qcc : version $(version) ] ;
- local command = [ common.get-invocation-command qcc : QCC : $(command) ] ;
- common.handle-options qcc : $(condition) : $(command) : $(options) ;
-}
-
-
-generators.register-c-compiler qcc.compile.c++ : CPP : OBJ : <toolset>qcc ;
-generators.register-c-compiler qcc.compile.c : C : OBJ : <toolset>qcc ;
-generators.register-c-compiler qcc.compile.asm : ASM : OBJ : <toolset>qcc ;
-
-
-# Declare flags for compilation.
-toolset.flags qcc.compile OPTIONS <debug-symbols>on : -gstabs+ ;
-
-# Declare flags and action for compilation.
-toolset.flags qcc.compile OPTIONS <optimization>off : -O0 ;
-toolset.flags qcc.compile OPTIONS <optimization>speed : -O3 ;
-toolset.flags qcc.compile OPTIONS <optimization>space : -Os ;
-
-toolset.flags qcc.compile OPTIONS <inlining>off : -Wc,-fno-inline ;
-toolset.flags qcc.compile OPTIONS <inlining>on : -Wc,-Wno-inline ;
-toolset.flags qcc.compile OPTIONS <inlining>full : -Wc,-finline-functions -Wc,-Wno-inline ;
-
-toolset.flags qcc.compile OPTIONS <warnings>off : -w ;
-toolset.flags qcc.compile OPTIONS <warnings>all : -Wc,-Wall ;
-toolset.flags qcc.compile OPTIONS <warnings-as-errors>on : -Wc,-Werror ;
-
-toolset.flags qcc.compile OPTIONS <profiling>on : -p ;
-
-toolset.flags qcc.compile OPTIONS <cflags> ;
-toolset.flags qcc.compile.c++ OPTIONS <cxxflags> ;
-toolset.flags qcc.compile DEFINES <define> ;
-toolset.flags qcc.compile INCLUDES <include> ;
-
-toolset.flags qcc.compile OPTIONS <link>shared : -shared ;
-
-toolset.flags qcc.compile.c++ TEMPLATE_DEPTH <c++-template-depth> ;
-
-
-rule compile.c++
-{
- # Here we want to raise the template-depth parameter value to something
- # higher than the default value of 17. Note that we could do this using the
- # feature.set-default rule but we do not want to set the default value for
- # all toolsets as well.
- #
- # TODO: This 'modified default' has been inherited from some 'older Boost
- # Build implementation' and has most likely been added to make some Boost
- # library parts compile correctly. We should see what exactly prompted this
- # and whether we can get around the problem more locally.
- local template-depth = [ on $(1) return $(TEMPLATE_DEPTH) ] ;
- if ! $(template-depth)
- {
- TEMPLATE_DEPTH on $(1) = 128 ;
- }
-}
-
-actions compile.c++
-{
- "$(CONFIG_COMMAND)" -Wc,-ftemplate-depth-$(TEMPLATE_DEPTH) $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
-}
-
-actions compile.c
-{
- "$(CONFIG_COMMAND)" $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
-}
-
-actions compile.asm
-{
- "$(CONFIG_COMMAND)" $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
-}
-
-
-# The class checking that we do not try to use the <runtime-link>static property
-# while creating or using a shared library, since it is not supported by qcc/
-# /libc.
-#
-class qcc-linking-generator : unix-linking-generator
-{
- rule generated-targets ( sources + : property-set : project name ? )
- {
- if <runtime-link>static in [ $(property-set).raw ]
- {
- local m ;
- if [ id ] = "qcc.link.dll"
- {
- m = "on qcc, DLL can't be build with <runtime-link>static" ;
- }
- if ! $(m)
- {
- for local s in $(sources)
- {
- local type = [ $(s).type ] ;
- if $(type) && [ type.is-derived $(type) SHARED_LIB ]
- {
- m = "on qcc, using DLLS together with the <runtime-link>static options is not possible " ;
- }
- }
- }
- if $(m)
- {
- errors.user-error $(m) : "It is suggested to use"
- "<runtime-link>static together with <link>static." ;
- }
- }
-
- return [ unix-linking-generator.generated-targets
- $(sources) : $(property-set) : $(project) $(name) ] ;
- }
-}
-
-generators.register [ new qcc-linking-generator qcc.link : LIB OBJ : EXE
- : <toolset>qcc ] ;
-
-generators.register [ new qcc-linking-generator qcc.link.dll : LIB OBJ
- : SHARED_LIB : <toolset>qcc ] ;
-
-generators.override qcc.prebuilt : builtin.prebuilt ;
-generators.override qcc.searched-lib-generator : searched-lib-generator ;
-
-
-# Declare flags for linking.
-# First, the common flags.
-toolset.flags qcc.link OPTIONS <debug-symbols>on : -gstabs+ ;
-toolset.flags qcc.link OPTIONS <profiling>on : -p ;
-toolset.flags qcc.link OPTIONS <linkflags> ;
-toolset.flags qcc.link LINKPATH <library-path> ;
-toolset.flags qcc.link FINDLIBS-ST <find-static-library> ;
-toolset.flags qcc.link FINDLIBS-SA <find-shared-library> ;
-toolset.flags qcc.link LIBRARIES <library-file> ;
-
-toolset.flags qcc.link FINDLIBS-SA : m ;
-
-# For <runtime-link>static we made sure there are no dynamic libraries in the
-# link.
-toolset.flags qcc.link OPTIONS <runtime-link>static : -static ;
-
-# Assuming this is just like with gcc.
-toolset.flags qcc.link RPATH : <dll-path> : unchecked ;
-toolset.flags qcc.link RPATH_LINK : <xdll-path> : unchecked ;
-
-
-# Declare actions for linking.
-#
-rule link ( targets * : sources * : properties * )
-{
- SPACE on $(targets) = " " ;
- # Serialize execution of the 'link' action, since running N links in
- # parallel is just slower. For now, serialize only qcc links while it might
- # be a good idea to serialize all links.
- JAM_SEMAPHORE on $(targets) = <s>qcc-link-semaphore ;
-}
-
-actions link bind LIBRARIES
-{
- "$(CONFIG_COMMAND)" -L"$(LINKPATH)" -Wl,-R$(SPACE)-Wl,"$(RPATH)" -Wl,-rpath-link$(SPACE)-Wl,"$(RPATH_LINK)" -o "$(<)" "$(>)" "$(LIBRARIES)" -l$(FINDLIBS-ST) -l$(FINDLIBS-SA) $(OPTIONS)
-}
-
-
-# Always remove archive and start again. Here is the rationale from Andre Hentz:
-# I had a file, say a1.c, that was included into liba.a. I moved a1.c to a2.c,
-# updated my Jamfiles and rebuilt. My program was crashing with absurd errors.
-# After some debugging I traced it back to the fact that a1.o was *still* in
-# liba.a
-RM = [ common.rm-command ] ;
-if [ os.name ] = NT
-{
- RM = "if exist \"$(<[1])\" DEL \"$(<[1])\"" ;
-}
-
-
-# Declare action for creating static libraries. The 'r' letter means to add
-# files to the archive with replacement. Since we remove the archive, we do not
-# care about replacement, but there is no option to "add without replacement".
-# The 'c' letter suppresses warnings in case the archive does not exists yet.
-# That warning is produced only on some platforms, for whatever reasons.
-#
-# Use qcc driver to create archive, see
-# http://www.qnx.com/developers/docs/6.3.2/neutrino/utilities/q/qcc.html
-actions piecemeal archive
-{
- $(RM) "$(<)"
- "$(CONFIG_COMMAND)" -A "$(<)" "$(>)"
-}
-
-
-rule link.dll ( targets * : sources * : properties * )
-{
- SPACE on $(targets) = " " ;
- JAM_SEMAPHORE on $(targets) = <s>qcc-link-semaphore ;
-}
-
-
-# Differ from 'link' above only by -shared.
-#
-actions link.dll bind LIBRARIES
-{
- "$(CONFIG_COMMAND)" -L"$(LINKPATH)" -Wl,-R$(SPACE)-Wl,"$(RPATH)" -o "$(<)" $(HAVE_SONAME)-Wl,-h$(SPACE)-Wl,$(<[1]:D=) -shared "$(>)" "$(LIBRARIES)" -l$(FINDLIBS-ST) -l$(FINDLIBS-SA) $(OPTIONS)
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/qt.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/qt.jam b/ext/kenlm/jam-files/boost-build/tools/qt.jam
deleted file mode 100644
index 8aa7ca2..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/qt.jam
+++ /dev/null
@@ -1,17 +0,0 @@
-# Copyright (c) 2006 Vladimir Prus.
-#
-# Use, modification and distribution is subject to the Boost Software
-# License Version 1.0. (See accompanying file LICENSE_1_0.txt or
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# Forwarning toolset file to Qt GUI library. Forwards to the toolset file
-# for the current version of Qt.
-
-import qt4 ;
-
-rule init ( prefix : full_bin ? : full_inc ? : full_lib ? : version ? : condition * )
-{
- qt4.init $(prefix) : $(full_bin) : $(full_inc) : $(full_lib) : $(version) : $(condition) ;
-}
-
-
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/tools/qt3.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/tools/qt3.jam b/ext/kenlm/jam-files/boost-build/tools/qt3.jam
deleted file mode 100644
index f82cf0a..0000000
--- a/ext/kenlm/jam-files/boost-build/tools/qt3.jam
+++ /dev/null
@@ -1,209 +0,0 @@
-# Copyright 2006 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-# Support for the Qt GUI library version 3
-# (http://www.trolltech.com/products/qt3/index.html).
-# For new developments, it is recommended to use Qt4 via the qt4 Boost.Build
-# module.
-
-import modules ;
-import feature ;
-import errors ;
-import type ;
-import "class" : new ;
-import generators ;
-import project ;
-import toolset : flags ;
-
-# Convert this module into a project, so that we can declare targets here.
-project.initialize $(__name__) ;
-project qt3 ;
-
-
-# Initialized the QT support module. The 'prefix' parameter tells where QT is
-# installed. When not given, environmental variable QTDIR should be set.
-#
-rule init ( prefix ? )
-{
- if ! $(prefix)
- {
- prefix = [ modules.peek : QTDIR ] ;
- if ! $(prefix)
- {
- errors.error
- "QT installation prefix not given and QTDIR variable is empty" ;
- }
- }
-
- if $(.initialized)
- {
- if $(prefix) != $(.prefix)
- {
- errors.error
- "Attempt the reinitialize QT with different installation prefix" ;
- }
- }
- else
- {
- .initialized = true ;
- .prefix = $(prefix) ;
-
- generators.register-standard qt3.moc : H : CPP(moc_%) : <allow>qt3 ;
- # Note: the OBJ target type here is fake, take a look at
- # qt4.jam/uic-h-generator for explanations that apply in this case as
- # well.
- generators.register [ new moc-h-generator-qt3
- qt3.moc.cpp : MOCCABLE_CPP : OBJ : <allow>qt3 ] ;
-
- # The UI type is defined in types/qt.jam, and UIC_H is only used in
- # qt.jam, but not in qt4.jam, so define it here.
- type.register UIC_H : : H ;
-
- generators.register-standard qt3.uic-h : UI : UIC_H : <allow>qt3 ;
-
- # The following generator is used to convert UI files to CPP. It creates
- # UIC_H from UI, and constructs CPP from UI/UIC_H. In addition, it also
- # returns UIC_H target, so that it can be mocced.
- class qt::uic-cpp-generator : generator
- {
- rule __init__ ( )
- {
- generator.__init__ qt3.uic-cpp : UI UIC_H : CPP : <allow>qt3 ;
- }
-
- rule run ( project name ? : properties * : sources + )
- {
- # Consider this:
- # obj test : test_a.cpp : <optimization>off ;
- #
- # This generator will somehow be called in this case, and,
- # will fail -- which is okay. However, if there are <library>
- # properties they will be converted to sources, so the size of
- # 'sources' will be more than 1. In this case, the base generator
- # will just crash -- and that's not good. Just use a quick test
- # here.
-
- local result ;
- if ! $(sources[2])
- {
- # Construct CPP as usual
- result = [ generator.run $(project) $(name)
- : $(properties) : $(sources) ] ;
-
- # If OK, process UIC_H with moc. It's pretty clear that
- # the object generated with UIC will have Q_OBJECT macro.
- if $(result)
- {
- local action = [ $(result[1]).action ] ;
- local sources = [ $(action).sources ] ;
- local mocced = [ generators.construct $(project) $(name)
- : CPP : $(properties) : $(sources[2]) ] ;
- result += $(mocced[2-]) ;
- }
- }
-
- return $(result) ;
- }
- }
-
- generators.register [ new qt::uic-cpp-generator ] ;
-
- # Finally, declare prebuilt target for QT library.
- local usage-requirements =
- <include>$(.prefix)/include
- <dll-path>$(.prefix)/lib
- <library-path>$(.prefix)/lib
- <allow>qt3
- ;
- lib qt : : <name>qt-mt <threading>multi : : $(usage-requirements) ;
- lib qt : : <name>qt <threading>single : : $(usage-requirements) ;
- }
-}
-
-class moc-h-generator-qt3 : generator
-{
- rule __init__ ( * : * )
- {
- generator.__init__ $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ;
- }
-
- rule run ( project name ? : property-set : sources * )
- {
- if ! $(sources[2]) && [ $(sources[1]).type ] = MOCCABLE_CPP
- {
- name = [ $(sources[1]).name ] ;
- name = $(name:B) ;
-
- local a = [ new action $(sources[1]) : qt3.moc.cpp :
- $(property-set) ] ;
-
- local target = [
- new file-target $(name) : MOC : $(project) : $(a) ] ;
-
- local r = [ virtual-target.register $(target) ] ;
-
- # Since this generator will return a H target, the linking generator
- # won't use it at all, and won't set any dependency on it. However,
- # we need the target to be seen by bjam, so that the dependency from
- # sources to this generated header is detected -- if Jam does not
- # know about this target, it won't do anything.
- DEPENDS all : [ $(r).actualize ] ;
-
- return $(r) ;
- }
- }
-}
-
-
-# Query the installation directory. This is needed in at least two scenarios.
-# First, when re-using sources from the Qt-Tree. Second, to "install" custom Qt
-# plugins to the Qt-Tree.
-#
-rule directory
-{
- return $(.prefix) ;
-}
-
-# -f forces moc to include the processed source file. Without it, it would think
-# that .qpp is not a header and would not include it from the generated file.
-#
-actions moc
-{
- $(.prefix)/bin/moc -f $(>) -o $(<)
-}
-
-# When moccing .cpp files, we don't need -f, otherwise generated code will
-# include .cpp and we'll get duplicated symbols.
-#
-actions moc.cpp
-{
- $(.prefix)/bin/moc $(>) -o $(<)
-}
-
-
-space = " " ;
-
-# Sometimes it's required to make 'plugins' available during uic invocation. To
-# help with this we add paths to all dependency libraries to uic commane line.
-# The intention is that it's possible to write
-#
-# exe a : ... a.ui ... : <uses>some_plugin ;
-#
-# and have everything work. We'd add quite a bunch of unrelated paths but it
-# won't hurt.
-#
-flags qt3.uic-h LIBRARY_PATH <xdll-path> ;
-actions uic-h
-{
- $(.prefix)/bin/uic $(>) -o $(<) -L$(space)$(LIBRARY_PATH)
-}
-
-
-flags qt3.uic-cpp LIBRARY_PATH <xdll-path> ;
-# The second target is uic-generated header name. It's placed in build dir, but
-# we want to include it using only basename.
-actions uic-cpp
-{
- $(.prefix)/bin/uic $(>[1]) -i $(>[2]:D=) -o $(<) -L$(space)$(LIBRARY_PATH)
-}
[51/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
Converted KenLM into a submodule
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/6da3961b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/6da3961b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/6da3961b
Branch: refs/heads/master
Commit: 6da3961bee52d10d17d9218ba267a505d1fb70ea
Parents: 76bb8fd
Author: Matt Post <po...@cs.jhu.edu>
Authored: Tue Apr 19 15:23:42 2016 -0400
Committer: Matt Post <po...@cs.jhu.edu>
Committed: Tue Apr 19 15:23:42 2016 -0400
----------------------------------------------------------------------
.gitmodules | 4 +
build.xml | 2 +-
ext/kenlm | 1 +
ext/kenlm/.gitignore | 32 -
ext/kenlm/BUILDING | 10 -
ext/kenlm/CMakeLists.txt | 57 -
ext/kenlm/COPYING | 502 --
ext/kenlm/COPYING.3 | 674 --
ext/kenlm/COPYING.LESSER.3 | 165 -
ext/kenlm/Doxyfile | 1519 ----
ext/kenlm/Jamroot | 68 -
ext/kenlm/LICENSE | 26 -
ext/kenlm/README.md | 96 -
ext/kenlm/bjam | 23 -
ext/kenlm/clean_query_only.sh | 2 -
ext/kenlm/cmake/KenLMFunctions.cmake | 82 -
ext/kenlm/cmake/modules/FindEigen3.cmake | 90 -
ext/kenlm/compile_query_only.sh | 34 -
ext/kenlm/jam-files/LICENSE_1_0.txt | 23 -
ext/kenlm/jam-files/boost-build/boost-build.jam | 8 -
ext/kenlm/jam-files/boost-build/bootstrap.jam | 18 -
.../jam-files/boost-build/build-system.jam | 981 ---
ext/kenlm/jam-files/boost-build/build/ac.jam | 303 -
ext/kenlm/jam-files/boost-build/build/alias.jam | 74 -
.../boost-build/build/build-request.jam | 322 -
.../boost-build/build/config-cache.jam | 64 -
.../jam-files/boost-build/build/configure.jam | 281 -
.../jam-files/boost-build/build/feature.jam | 1350 ----
.../jam-files/boost-build/build/generators.jam | 1420 ----
.../jam-files/boost-build/build/project.jam | 1228 ---
.../boost-build/build/property-set.jam | 517 --
.../jam-files/boost-build/build/property.jam | 905 ---
.../jam-files/boost-build/build/readme.txt | 11 -
.../jam-files/boost-build/build/scanner.jam | 163 -
.../jam-files/boost-build/build/targets.jam | 1694 ----
.../jam-files/boost-build/build/toolset.jam | 582 --
ext/kenlm/jam-files/boost-build/build/type.jam | 401 -
.../jam-files/boost-build/build/version.jam | 165 -
.../boost-build/build/virtual-target.jam | 1339 ----
.../boost-build/kernel/boost-build.jam | 5 -
.../jam-files/boost-build/kernel/bootstrap.jam | 266 -
.../jam-files/boost-build/kernel/class.jam | 420 -
.../jam-files/boost-build/kernel/errors.jam | 287 -
.../jam-files/boost-build/kernel/modules.jam | 359 -
.../jam-files/boost-build/options/help.jam | 212 -
ext/kenlm/jam-files/boost-build/site-config.jam | 11 -
ext/kenlm/jam-files/boost-build/tools/acc.jam | 118 -
.../jam-files/boost-build/tools/auto-index.jam | 204 -
ext/kenlm/jam-files/boost-build/tools/bison.jam | 32 -
.../boost-build/tools/boostbook-config.jam | 13 -
.../jam-files/boost-build/tools/boostbook.jam | 773 --
.../jam-files/boost-build/tools/borland.jam | 220 -
.../jam-files/boost-build/tools/builtin.jam | 974 ---
ext/kenlm/jam-files/boost-build/tools/cast.jam | 91 -
.../boost-build/tools/clang-darwin.jam | 170 -
.../jam-files/boost-build/tools/clang-linux.jam | 196 -
ext/kenlm/jam-files/boost-build/tools/clang.jam | 27 -
.../jam-files/boost-build/tools/common.jam | 988 ---
.../jam-files/boost-build/tools/como-linux.jam | 103 -
.../jam-files/boost-build/tools/como-win.jam | 117 -
ext/kenlm/jam-files/boost-build/tools/como.jam | 29 -
.../jam-files/boost-build/tools/convert.jam | 62 -
ext/kenlm/jam-files/boost-build/tools/cray.jam | 115 -
.../jam-files/boost-build/tools/cw-config.jam | 34 -
ext/kenlm/jam-files/boost-build/tools/cw.jam | 246 -
.../jam-files/boost-build/tools/darwin.jam | 592 --
ext/kenlm/jam-files/boost-build/tools/dmc.jam | 134 -
.../jam-files/boost-build/tools/docutils.jam | 85 -
.../boost-build/tools/doxygen-config.jam | 11 -
.../jam-files/boost-build/tools/doxygen.jam | 775 --
.../tools/doxygen/windows-paths-check.doxyfile | 3 -
.../tools/doxygen/windows-paths-check.hpp | 0
ext/kenlm/jam-files/boost-build/tools/fop.jam | 69 -
.../jam-files/boost-build/tools/fortran.jam | 55 -
ext/kenlm/jam-files/boost-build/tools/gcc.jam | 1184 ---
.../jam-files/boost-build/tools/generate.jam | 108 -
.../jam-files/boost-build/tools/gettext.jam | 230 -
.../jam-files/boost-build/tools/gfortran.jam | 39 -
.../jam-files/boost-build/tools/hp_cxx.jam | 181 -
.../jam-files/boost-build/tools/hpfortran.jam | 35 -
ext/kenlm/jam-files/boost-build/tools/ifort.jam | 44 -
.../boost-build/tools/intel-darwin.jam | 227 -
.../jam-files/boost-build/tools/intel-linux.jam | 250 -
.../jam-files/boost-build/tools/intel-win.jam | 184 -
ext/kenlm/jam-files/boost-build/tools/intel.jam | 34 -
ext/kenlm/jam-files/boost-build/tools/jpeg.jam | 233 -
ext/kenlm/jam-files/boost-build/tools/lex.jam | 33 -
ext/kenlm/jam-files/boost-build/tools/make.jam | 63 -
ext/kenlm/jam-files/boost-build/tools/mc.jam | 44 -
.../jam-files/boost-build/tools/message.jam | 62 -
ext/kenlm/jam-files/boost-build/tools/midl.jam | 142 -
.../jam-files/boost-build/tools/mipspro.jam | 145 -
ext/kenlm/jam-files/boost-build/tools/mpi.jam | 583 --
.../jam-files/boost-build/tools/msvc-config.jam | 12 -
ext/kenlm/jam-files/boost-build/tools/msvc.jam | 1435 ----
.../jam-files/boost-build/tools/notfile.jam | 65 -
.../jam-files/boost-build/tools/package.jam | 165 -
.../jam-files/boost-build/tools/pathscale.jam | 178 -
ext/kenlm/jam-files/boost-build/tools/pch.jam | 95 -
ext/kenlm/jam-files/boost-build/tools/pgi.jam | 147 -
ext/kenlm/jam-files/boost-build/tools/png.jam | 226 -
.../boost-build/tools/python-config.jam | 27 -
.../jam-files/boost-build/tools/python.jam | 1258 ---
ext/kenlm/jam-files/boost-build/tools/qcc.jam | 238 -
ext/kenlm/jam-files/boost-build/tools/qt.jam | 17 -
ext/kenlm/jam-files/boost-build/tools/qt3.jam | 209 -
ext/kenlm/jam-files/boost-build/tools/qt4.jam | 755 --
ext/kenlm/jam-files/boost-build/tools/qt5.jam | 728 --
.../boost-build/tools/quickbook-config.jam | 44 -
.../jam-files/boost-build/tools/quickbook.jam | 361 -
ext/kenlm/jam-files/boost-build/tools/rc.jam | 155 -
ext/kenlm/jam-files/boost-build/tools/stage.jam | 519 --
.../jam-files/boost-build/tools/stlport.jam | 309 -
ext/kenlm/jam-files/boost-build/tools/sun.jam | 142 -
.../jam-files/boost-build/tools/symlink.jam | 140 -
.../jam-files/boost-build/tools/testing-aux.jam | 220 -
.../jam-files/boost-build/tools/testing.jam | 594 --
ext/kenlm/jam-files/boost-build/tools/tiff.jam | 230 -
.../jam-files/boost-build/tools/types/asm.jam | 4 -
.../jam-files/boost-build/tools/types/cpp.jam | 90 -
.../jam-files/boost-build/tools/types/exe.jam | 9 -
.../jam-files/boost-build/tools/types/html.jam | 4 -
.../jam-files/boost-build/tools/types/lib.jam | 74 -
.../jam-files/boost-build/tools/types/obj.jam | 9 -
.../jam-files/boost-build/tools/types/objc.jam | 26 -
.../boost-build/tools/types/preprocessed.jam | 9 -
.../jam-files/boost-build/tools/types/qt.jam | 12 -
.../boost-build/tools/types/register.jam | 39 -
.../jam-files/boost-build/tools/types/rsp.jam | 4 -
ext/kenlm/jam-files/boost-build/tools/unix.jam | 224 -
ext/kenlm/jam-files/boost-build/tools/vacpp.jam | 150 -
ext/kenlm/jam-files/boost-build/tools/whale.jam | 116 -
ext/kenlm/jam-files/boost-build/tools/xlf.jam | 39 -
.../boost-build/tools/xsltproc-config.jam | 36 -
.../jam-files/boost-build/tools/xsltproc.jam | 205 -
.../boost-build/tools/xsltproc/included.xsl | 11 -
.../boost-build/tools/xsltproc/test.xml | 2 -
.../boost-build/tools/xsltproc/test.xsl | 12 -
ext/kenlm/jam-files/boost-build/tools/zlib.jam | 227 -
ext/kenlm/jam-files/boost-build/user-config.jam | 92 -
ext/kenlm/jam-files/boost-build/util/assert.jam | 346 -
.../jam-files/boost-build/util/container.jam | 339 -
ext/kenlm/jam-files/boost-build/util/doc.jam | 1002 ---
.../jam-files/boost-build/util/indirect.jam | 117 -
.../jam-files/boost-build/util/numbers.jam | 218 -
ext/kenlm/jam-files/boost-build/util/option.jam | 109 -
ext/kenlm/jam-files/boost-build/util/order.jam | 169 -
ext/kenlm/jam-files/boost-build/util/os.jam | 171 -
ext/kenlm/jam-files/boost-build/util/path.jam | 910 ---
ext/kenlm/jam-files/boost-build/util/print.jam | 488 --
ext/kenlm/jam-files/boost-build/util/regex.jam | 203 -
.../jam-files/boost-build/util/sequence.jam | 339 -
ext/kenlm/jam-files/boost-build/util/set.jam | 93 -
ext/kenlm/jam-files/boost-build/util/string.jam | 189 -
.../jam-files/boost-build/util/utility.jam | 235 -
ext/kenlm/jam-files/empty_test_main.cc | 8 -
ext/kenlm/jam-files/engine/Jambase | 2473 ------
ext/kenlm/jam-files/engine/boost-jam.spec | 64 -
ext/kenlm/jam-files/engine/boost-no-inspect | 1 -
ext/kenlm/jam-files/engine/build.bat | 590 --
ext/kenlm/jam-files/engine/build.jam | 1022 ---
ext/kenlm/jam-files/engine/build.sh | 303 -
ext/kenlm/jam-files/engine/builtins.c | 2354 ------
ext/kenlm/jam-files/engine/builtins.h | 70 -
ext/kenlm/jam-files/engine/bump_version.py | 98 -
ext/kenlm/jam-files/engine/class.c | 191 -
ext/kenlm/jam-files/engine/class.h | 14 -
ext/kenlm/jam-files/engine/command.c | 84 -
ext/kenlm/jam-files/engine/command.h | 73 -
ext/kenlm/jam-files/engine/compile.c | 271 -
ext/kenlm/jam-files/engine/compile.h | 59 -
ext/kenlm/jam-files/engine/constants.c | 186 -
ext/kenlm/jam-files/engine/constants.h | 73 -
ext/kenlm/jam-files/engine/cwd.c | 83 -
ext/kenlm/jam-files/engine/cwd.h | 35 -
ext/kenlm/jam-files/engine/debian/changelog | 72 -
ext/kenlm/jam-files/engine/debian/control | 16 -
ext/kenlm/jam-files/engine/debian/copyright | 25 -
ext/kenlm/jam-files/engine/debian/jam.man.sgml | 236 -
ext/kenlm/jam-files/engine/debian/rules | 73 -
ext/kenlm/jam-files/engine/debug.c | 145 -
ext/kenlm/jam-files/engine/debug.h | 55 -
ext/kenlm/jam-files/engine/execcmd.c | 121 -
ext/kenlm/jam-files/engine/execcmd.h | 102 -
ext/kenlm/jam-files/engine/execnt.c | 1232 ---
ext/kenlm/jam-files/engine/execunix.c | 559 --
ext/kenlm/jam-files/engine/filent.c | 435 --
ext/kenlm/jam-files/engine/filesys.c | 326 -
ext/kenlm/jam-files/engine/filesys.h | 57 -
ext/kenlm/jam-files/engine/fileunix.c | 464 --
ext/kenlm/jam-files/engine/frames.c | 29 -
ext/kenlm/jam-files/engine/frames.h | 42 -
ext/kenlm/jam-files/engine/function.c | 4870 ------------
ext/kenlm/jam-files/engine/function.h | 46 -
ext/kenlm/jam-files/engine/glob.c | 152 -
ext/kenlm/jam-files/engine/hash.c | 387 -
ext/kenlm/jam-files/engine/hash.h | 79 -
ext/kenlm/jam-files/engine/hcache.c | 519 --
ext/kenlm/jam-files/engine/hcache.h | 19 -
ext/kenlm/jam-files/engine/hdrmacro.c | 139 -
ext/kenlm/jam-files/engine/hdrmacro.h | 21 -
ext/kenlm/jam-files/engine/headers.c | 197 -
ext/kenlm/jam-files/engine/headers.h | 25 -
ext/kenlm/jam-files/engine/jam.c | 656 --
ext/kenlm/jam-files/engine/jam.h | 475 --
ext/kenlm/jam-files/engine/jambase.c | 1691 ----
ext/kenlm/jam-files/engine/jambase.h | 15 -
ext/kenlm/jam-files/engine/jamgram.c | 2327 ------
ext/kenlm/jam-files/engine/jamgram.h | 149 -
ext/kenlm/jam-files/engine/jamgram.y | 373 -
ext/kenlm/jam-files/engine/jamgram.yy | 329 -
ext/kenlm/jam-files/engine/jamgramtab.h | 44 -
ext/kenlm/jam-files/engine/lists.c | 475 --
ext/kenlm/jam-files/engine/lists.h | 113 -
ext/kenlm/jam-files/engine/make.c | 908 ---
ext/kenlm/jam-files/engine/make.h | 44 -
ext/kenlm/jam-files/engine/make1.c | 1283 ---
ext/kenlm/jam-files/engine/md5.c | 381 -
ext/kenlm/jam-files/engine/md5.h | 91 -
ext/kenlm/jam-files/engine/mem.c | 75 -
ext/kenlm/jam-files/engine/mem.h | 133 -
ext/kenlm/jam-files/engine/mkjambase.c | 123 -
ext/kenlm/jam-files/engine/modules.c | 431 -
ext/kenlm/jam-files/engine/modules.h | 52 -
ext/kenlm/jam-files/engine/modules/order.c | 160 -
ext/kenlm/jam-files/engine/modules/path.c | 25 -
.../jam-files/engine/modules/property-set.c | 330 -
ext/kenlm/jam-files/engine/modules/readme.txt | 3 -
ext/kenlm/jam-files/engine/modules/regex.c | 220 -
ext/kenlm/jam-files/engine/modules/sequence.c | 97 -
ext/kenlm/jam-files/engine/modules/set.c | 43 -
ext/kenlm/jam-files/engine/native.c | 34 -
ext/kenlm/jam-files/engine/native.h | 34 -
ext/kenlm/jam-files/engine/object.c | 394 -
ext/kenlm/jam-files/engine/object.h | 44 -
ext/kenlm/jam-files/engine/option.c | 94 -
ext/kenlm/jam-files/engine/option.h | 23 -
ext/kenlm/jam-files/engine/output.c | 98 -
ext/kenlm/jam-files/engine/output.h | 30 -
ext/kenlm/jam-files/engine/parse.c | 132 -
ext/kenlm/jam-files/engine/parse.h | 76 -
ext/kenlm/jam-files/engine/patchlevel.h | 17 -
ext/kenlm/jam-files/engine/pathnt.c | 301 -
ext/kenlm/jam-files/engine/pathsys.c | 285 -
ext/kenlm/jam-files/engine/pathsys.h | 85 -
ext/kenlm/jam-files/engine/pathunix.c | 71 -
ext/kenlm/jam-files/engine/regexp.c | 1329 ----
ext/kenlm/jam-files/engine/regexp.h | 34 -
ext/kenlm/jam-files/engine/rules.c | 740 --
ext/kenlm/jam-files/engine/rules.h | 270 -
ext/kenlm/jam-files/engine/scan.c | 404 -
ext/kenlm/jam-files/engine/scan.h | 61 -
ext/kenlm/jam-files/engine/search.c | 274 -
ext/kenlm/jam-files/engine/search.h | 22 -
ext/kenlm/jam-files/engine/strings.c | 223 -
ext/kenlm/jam-files/engine/strings.h | 36 -
ext/kenlm/jam-files/engine/subst.c | 116 -
ext/kenlm/jam-files/engine/subst.h | 14 -
ext/kenlm/jam-files/engine/timestamp.c | 262 -
ext/kenlm/jam-files/engine/timestamp.h | 47 -
ext/kenlm/jam-files/engine/variable.c | 345 -
ext/kenlm/jam-files/engine/variable.h | 34 -
ext/kenlm/jam-files/engine/w32_getreg.c | 201 -
ext/kenlm/jam-files/engine/yyacc.c | 268 -
ext/kenlm/jam-files/fail/Jamroot | 4 -
ext/kenlm/jam-files/sanity.jam | 344 -
ext/kenlm/lm/CMakeLists.txt | 90 -
ext/kenlm/lm/Jamfile | 40 -
ext/kenlm/lm/bhiksha.cc | 94 -
ext/kenlm/lm/bhiksha.hh | 122 -
ext/kenlm/lm/binary_format.cc | 302 -
ext/kenlm/lm/binary_format.hh | 106 -
ext/kenlm/lm/blank.hh | 42 -
ext/kenlm/lm/build_binary_main.cc | 234 -
ext/kenlm/lm/builder/CMakeLists.txt | 67 -
ext/kenlm/lm/builder/Jamfile | 13 -
ext/kenlm/lm/builder/README.md | 47 -
ext/kenlm/lm/builder/TODO | 5 -
ext/kenlm/lm/builder/adjust_counts.cc | 353 -
ext/kenlm/lm/builder/adjust_counts.hh | 72 -
ext/kenlm/lm/builder/adjust_counts_test.cc | 112 -
ext/kenlm/lm/builder/combine_counts.hh | 31 -
ext/kenlm/lm/builder/corpus_count.cc | 239 -
ext/kenlm/lm/builder/corpus_count.hh | 53 -
ext/kenlm/lm/builder/corpus_count_test.cc | 79 -
ext/kenlm/lm/builder/debug_print.hh | 70 -
ext/kenlm/lm/builder/discount.hh | 26 -
ext/kenlm/lm/builder/dump_counts_main.cc | 36 -
ext/kenlm/lm/builder/hash_gamma.hh | 19 -
ext/kenlm/lm/builder/header_info.hh | 28 -
ext/kenlm/lm/builder/initial_probabilities.cc | 306 -
ext/kenlm/lm/builder/initial_probabilities.hh | 45 -
ext/kenlm/lm/builder/interpolate.cc | 166 -
ext/kenlm/lm/builder/interpolate.hh | 37 -
ext/kenlm/lm/builder/lmplz_main.cc | 220 -
ext/kenlm/lm/builder/output.cc | 52 -
ext/kenlm/lm/builder/output.hh | 85 -
ext/kenlm/lm/builder/payload.hh | 48 -
ext/kenlm/lm/builder/pipeline.cc | 385 -
ext/kenlm/lm/builder/pipeline.hh | 76 -
ext/kenlm/lm/common/CMakeLists.txt | 40 -
ext/kenlm/lm/common/Jamfile | 2 -
ext/kenlm/lm/common/compare.hh | 174 -
ext/kenlm/lm/common/joint_order.hh | 71 -
ext/kenlm/lm/common/model_buffer.cc | 91 -
ext/kenlm/lm/common/model_buffer.hh | 63 -
ext/kenlm/lm/common/ngram.hh | 77 -
ext/kenlm/lm/common/ngram_stream.hh | 65 -
ext/kenlm/lm/common/print.cc | 62 -
ext/kenlm/lm/common/print.hh | 58 -
ext/kenlm/lm/common/renumber.cc | 17 -
ext/kenlm/lm/common/renumber.hh | 30 -
ext/kenlm/lm/common/size_option.cc | 24 -
ext/kenlm/lm/common/size_option.hh | 11 -
ext/kenlm/lm/common/special.hh | 27 -
ext/kenlm/lm/config.cc | 30 -
ext/kenlm/lm/config.hh | 124 -
ext/kenlm/lm/enumerate_vocab.hh | 28 -
ext/kenlm/lm/facade.hh | 73 -
ext/kenlm/lm/filter/CMakeLists.txt | 62 -
ext/kenlm/lm/filter/Jamfile | 7 -
ext/kenlm/lm/filter/arpa_io.cc | 77 -
ext/kenlm/lm/filter/arpa_io.hh | 99 -
ext/kenlm/lm/filter/count_io.hh | 89 -
ext/kenlm/lm/filter/filter_main.cc | 253 -
ext/kenlm/lm/filter/format.hh | 250 -
ext/kenlm/lm/filter/phrase.cc | 292 -
ext/kenlm/lm/filter/phrase.hh | 168 -
ext/kenlm/lm/filter/phrase_table_vocab_main.cc | 165 -
ext/kenlm/lm/filter/thread.hh | 167 -
ext/kenlm/lm/filter/vocab.cc | 53 -
ext/kenlm/lm/filter/vocab.hh | 133 -
ext/kenlm/lm/filter/wrapper.hh | 56 -
ext/kenlm/lm/fragment_main.cc | 37 -
ext/kenlm/lm/interpolate/CMakeLists.txt | 54 -
ext/kenlm/lm/interpolate/Jamfile | 22 -
ext/kenlm/lm/interpolate/backoff_matrix.hh | 29 -
.../lm/interpolate/backoff_reunification.cc | 57 -
.../lm/interpolate/backoff_reunification.hh | 27 -
.../interpolate/backoff_reunification_test.cc | 159 -
.../lm/interpolate/bounded_sequence_encoding.cc | 36 -
.../lm/interpolate/bounded_sequence_encoding.hh | 76 -
.../bounded_sequence_encoding_test.cc | 86 -
.../lm/interpolate/enumerate_global_vocab.cc | 48 -
.../lm/interpolate/enumerate_global_vocab.hh | 38 -
ext/kenlm/lm/interpolate/interpolate_info.hh | 35 -
ext/kenlm/lm/interpolate/interpolate_main.cc | 37 -
ext/kenlm/lm/interpolate/merge_probabilities.cc | 285 -
ext/kenlm/lm/interpolate/merge_probabilities.hh | 89 -
ext/kenlm/lm/interpolate/merge_test/test1 | Bin 26 -> 0 bytes
ext/kenlm/lm/interpolate/merge_test/test2 | Bin 39 -> 0 bytes
ext/kenlm/lm/interpolate/merge_test/test3 | Bin 16 -> 0 bytes
.../lm/interpolate/merge_test/test_bad_order | Bin 14 -> 0 bytes
ext/kenlm/lm/interpolate/merge_test/test_no_unk | 1 -
ext/kenlm/lm/interpolate/merge_vocab.cc | 131 -
ext/kenlm/lm/interpolate/merge_vocab.hh | 23 -
ext/kenlm/lm/interpolate/merge_vocab_test.cc | 126 -
ext/kenlm/lm/interpolate/normalize.cc | 384 -
ext/kenlm/lm/interpolate/normalize.hh | 35 -
ext/kenlm/lm/interpolate/normalize_test.cc | 86 -
ext/kenlm/lm/interpolate/perf_enum_gv_main.cc | 215 -
ext/kenlm/lm/interpolate/pipeline.cc | 159 -
ext/kenlm/lm/interpolate/pipeline.hh | 22 -
ext/kenlm/lm/interpolate/split_worker.cc | 40 -
ext/kenlm/lm/interpolate/split_worker.hh | 44 -
.../lm/interpolate/streaming_example_main.cc | 195 -
...ear_interpolation.lambda1_0.4.lambda2_0.6.lm | 23 -
...ear_interpolation.lambda1_0.4.lambda2_0.6.lm | 23 -
ext/kenlm/lm/interpolate/toy_data/toy1.lm | 23 -
ext/kenlm/lm/interpolate/toy_data/toy2.lm | 23 -
.../lm/interpolate/train_params_grant_main.cc | 561 --
ext/kenlm/lm/interpolate/train_params_main.cc | 349 -
ext/kenlm/lm/interpolate/tune_derivatives.cc | 91 -
ext/kenlm/lm/interpolate/tune_derivatives.hh | 30 -
.../lm/interpolate/tune_derivatives_test.cc | 89 -
ext/kenlm/lm/interpolate/tune_instance.cc | 354 -
ext/kenlm/lm/interpolate/tune_instance.hh | 86 -
.../interpolate/tune_instance_data/generate.sh | 9 -
.../lm/interpolate/tune_instance_data/toy0.1 | Bin 60 -> 0 bytes
.../lm/interpolate/tune_instance_data/toy0.2 | Bin 112 -> 0 bytes
.../lm/interpolate/tune_instance_data/toy0.3 | Bin 140 -> 0 bytes
.../tune_instance_data/toy0.kenlm_intermediate | 3 -
.../interpolate/tune_instance_data/toy0.vocab | Bin 19 -> 0 bytes
.../lm/interpolate/tune_instance_data/toy1.1 | Bin 72 -> 0 bytes
.../lm/interpolate/tune_instance_data/toy1.2 | Bin 112 -> 0 bytes
.../lm/interpolate/tune_instance_data/toy1.3 | Bin 120 -> 0 bytes
.../tune_instance_data/toy1.kenlm_intermediate | 3 -
.../interpolate/tune_instance_data/toy1.vocab | Bin 21 -> 0 bytes
ext/kenlm/lm/interpolate/tune_instance_test.cc | 93 -
ext/kenlm/lm/interpolate/tune_main.cc | 59 -
ext/kenlm/lm/interpolate/tune_matrix.hh | 17 -
ext/kenlm/lm/interpolate/universal_vocab.cc | 13 -
ext/kenlm/lm/interpolate/universal_vocab.hh | 38 -
ext/kenlm/lm/kenlm_benchmark_main.cc | 142 -
ext/kenlm/lm/left.hh | 216 -
ext/kenlm/lm/left_test.cc | 397 -
ext/kenlm/lm/lm_exception.cc | 23 -
ext/kenlm/lm/lm_exception.hh | 50 -
ext/kenlm/lm/max_order.hh | 13 -
ext/kenlm/lm/model.cc | 349 -
ext/kenlm/lm/model.hh | 155 -
ext/kenlm/lm/model_test.cc | 448 --
ext/kenlm/lm/model_type.hh | 23 -
ext/kenlm/lm/neural/Jamfile | 6 -
ext/kenlm/lm/neural/wordvecs.hh | 38 -
ext/kenlm/lm/ngram_query.hh | 113 -
ext/kenlm/lm/partial.hh | 166 -
ext/kenlm/lm/partial_test.cc | 199 -
ext/kenlm/lm/quantize.cc | 93 -
ext/kenlm/lm/quantize.hh | 233 -
ext/kenlm/lm/query_main.cc | 115 -
ext/kenlm/lm/read_arpa.cc | 161 -
ext/kenlm/lm/read_arpa.hh | 95 -
ext/kenlm/lm/return.hh | 42 -
ext/kenlm/lm/search_hashed.cc | 298 -
ext/kenlm/lm/search_hashed.hh | 192 -
ext/kenlm/lm/search_trie.cc | 600 --
ext/kenlm/lm/search_trie.hh | 129 -
ext/kenlm/lm/sizes.cc | 63 -
ext/kenlm/lm/sizes.hh | 17 -
ext/kenlm/lm/state.hh | 125 -
ext/kenlm/lm/test.arpa | 124 -
ext/kenlm/lm/test_nounk.arpa | 120 -
ext/kenlm/lm/trie.cc | 131 -
ext/kenlm/lm/trie.hh | 146 -
ext/kenlm/lm/trie_sort.cc | 304 -
ext/kenlm/lm/trie_sort.hh | 114 -
ext/kenlm/lm/value.hh | 158 -
ext/kenlm/lm/value_build.cc | 59 -
ext/kenlm/lm/value_build.hh | 97 -
ext/kenlm/lm/virtual_interface.cc | 19 -
ext/kenlm/lm/virtual_interface.hh | 160 -
ext/kenlm/lm/vocab.cc | 329 -
ext/kenlm/lm/vocab.hh | 279 -
ext/kenlm/lm/weights.hh | 22 -
ext/kenlm/lm/word_index.hh | 15 -
ext/kenlm/lm/wrappers/README | 3 -
ext/kenlm/lm/wrappers/nplm.cc | 116 -
ext/kenlm/lm/wrappers/nplm.hh | 85 -
ext/kenlm/python/_kenlm.pxd | 33 -
ext/kenlm/python/example.py | 28 -
ext/kenlm/python/kenlm.cpp | 7379 ------------------
ext/kenlm/python/kenlm.pyx | 231 -
ext/kenlm/setup.py | 48 -
ext/kenlm/util/CMakeLists.txt | 81 -
ext/kenlm/util/Jamfile | 41 -
ext/kenlm/util/bit_packing.cc | 40 -
ext/kenlm/util/bit_packing.hh | 185 -
ext/kenlm/util/bit_packing_test.cc | 59 -
ext/kenlm/util/cat_compressed_main.cc | 47 -
ext/kenlm/util/double-conversion/CMakeLists.txt | 39 -
ext/kenlm/util/double-conversion/Jamfile | 1 -
ext/kenlm/util/double-conversion/LICENSE | 26 -
ext/kenlm/util/double-conversion/bignum-dtoa.cc | 640 --
ext/kenlm/util/double-conversion/bignum-dtoa.h | 84 -
ext/kenlm/util/double-conversion/bignum.cc | 764 --
ext/kenlm/util/double-conversion/bignum.h | 145 -
.../util/double-conversion/cached-powers.cc | 175 -
.../util/double-conversion/cached-powers.h | 64 -
ext/kenlm/util/double-conversion/diy-fp.cc | 57 -
ext/kenlm/util/double-conversion/diy-fp.h | 118 -
.../util/double-conversion/double-conversion.cc | 889 ---
.../util/double-conversion/double-conversion.h | 536 --
ext/kenlm/util/double-conversion/fast-dtoa.cc | 664 --
ext/kenlm/util/double-conversion/fast-dtoa.h | 88 -
ext/kenlm/util/double-conversion/fixed-dtoa.cc | 402 -
ext/kenlm/util/double-conversion/fixed-dtoa.h | 56 -
ext/kenlm/util/double-conversion/ieee.h | 398 -
ext/kenlm/util/double-conversion/strtod.cc | 558 --
ext/kenlm/util/double-conversion/strtod.h | 45 -
ext/kenlm/util/double-conversion/utils.h | 320 -
ext/kenlm/util/ersatz_progress.cc | 47 -
ext/kenlm/util/ersatz_progress.hh | 57 -
ext/kenlm/util/exception.cc | 105 -
ext/kenlm/util/exception.hh | 159 -
ext/kenlm/util/fake_ostream.hh | 111 -
ext/kenlm/util/file.cc | 574 --
ext/kenlm/util/file.hh | 154 -
ext/kenlm/util/file_piece.cc | 337 -
ext/kenlm/util/file_piece.hh | 175 -
ext/kenlm/util/file_piece_test.cc | 154 -
ext/kenlm/util/file_stream.hh | 89 -
ext/kenlm/util/fixed_array.hh | 176 -
ext/kenlm/util/float_to_string.cc | 23 -
ext/kenlm/util/float_to_string.hh | 25 -
ext/kenlm/util/getopt.c | 78 -
ext/kenlm/util/getopt.hh | 33 -
ext/kenlm/util/have.hh | 13 -
ext/kenlm/util/integer_to_string.cc | 667 --
ext/kenlm/util/integer_to_string.hh | 66 -
ext/kenlm/util/integer_to_string_test.cc | 81 -
ext/kenlm/util/joint_sort.hh | 146 -
ext/kenlm/util/joint_sort_test.cc | 62 -
ext/kenlm/util/mmap.cc | 402 -
ext/kenlm/util/mmap.hh | 225 -
ext/kenlm/util/multi_intersection.hh | 80 -
ext/kenlm/util/multi_intersection_test.cc | 63 -
ext/kenlm/util/murmur_hash.cc | 175 -
ext/kenlm/util/murmur_hash.hh | 18 -
ext/kenlm/util/parallel_read.cc | 69 -
ext/kenlm/util/parallel_read.hh | 16 -
ext/kenlm/util/pcqueue.hh | 156 -
ext/kenlm/util/pcqueue_test.cc | 20 -
ext/kenlm/util/pool.cc | 38 -
ext/kenlm/util/pool.hh | 44 -
ext/kenlm/util/probing_hash_table.hh | 421 -
.../util/probing_hash_table_benchmark_main.cc | 181 -
ext/kenlm/util/probing_hash_table_test.cc | 102 -
ext/kenlm/util/proxy_iterator.hh | 101 -
ext/kenlm/util/read_compressed.cc | 448 --
ext/kenlm/util/read_compressed.hh | 84 -
ext/kenlm/util/read_compressed_test.cc | 130 -
ext/kenlm/util/scoped.cc | 43 -
ext/kenlm/util/scoped.hh | 111 -
ext/kenlm/util/sized_iterator.hh | 120 -
ext/kenlm/util/sized_iterator_test.cc | 16 -
ext/kenlm/util/sorted_uniform.hh | 105 -
ext/kenlm/util/sorted_uniform_test.cc | 127 -
ext/kenlm/util/stream/CMakeLists.txt | 50 -
ext/kenlm/util/stream/Jamfile | 7 -
ext/kenlm/util/stream/block.hh | 93 -
ext/kenlm/util/stream/chain.cc | 163 -
ext/kenlm/util/stream/chain.hh | 347 -
ext/kenlm/util/stream/config.hh | 63 -
ext/kenlm/util/stream/count_records.cc | 12 -
ext/kenlm/util/stream/count_records.hh | 20 -
ext/kenlm/util/stream/io.cc | 78 -
ext/kenlm/util/stream/io.hh | 87 -
ext/kenlm/util/stream/io_test.cc | 38 -
ext/kenlm/util/stream/line_input.cc | 52 -
ext/kenlm/util/stream/line_input.hh | 22 -
ext/kenlm/util/stream/multi_progress.cc | 86 -
ext/kenlm/util/stream/multi_progress.hh | 89 -
ext/kenlm/util/stream/multi_stream.hh | 124 -
ext/kenlm/util/stream/rewindable_stream.cc | 134 -
ext/kenlm/util/stream/rewindable_stream.hh | 132 -
ext/kenlm/util/stream/rewindable_stream_test.cc | 41 -
ext/kenlm/util/stream/sort.hh | 599 --
ext/kenlm/util/stream/sort_test.cc | 62 -
ext/kenlm/util/stream/stream.hh | 77 -
ext/kenlm/util/stream/stream_test.cc | 35 -
ext/kenlm/util/stream/timer.hh | 16 -
ext/kenlm/util/string_piece.cc | 192 -
ext/kenlm/util/string_piece.hh | 270 -
ext/kenlm/util/string_piece_hash.hh | 43 -
ext/kenlm/util/string_stream.hh | 44 -
ext/kenlm/util/string_stream_test.cc | 80 -
ext/kenlm/util/thread_pool.hh | 94 -
ext/kenlm/util/tokenize_piece.hh | 150 -
ext/kenlm/util/tokenize_piece_test.cc | 48 -
ext/kenlm/util/usage.cc | 305 -
ext/kenlm/util/usage.hh | 27 -
ext/kenlm/windows/build_binary.vcxproj | 174 -
ext/kenlm/windows/kenlm.sln | 64 -
ext/kenlm/windows/kenlm.vcxproj | 291 -
ext/kenlm/windows/lmplz.vcxproj | 214 -
ext/kenlm/windows/ngram_query.vcxproj | 177 -
jni/build_kenlm.sh | 2 +-
558 files changed, 7 insertions(+), 121077 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/.gitmodules
----------------------------------------------------------------------
diff --git a/.gitmodules b/.gitmodules
index 65e0ea5..b14568b 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -4,3 +4,7 @@
[submodule "berkeleylm"]
path = ext/berkeleylm
url = https://github.com/joshua-decoder/berkeleylm.git
+[submodule "ext/kenlm"]
+ path = ext/kenlm
+ url = https://github.com/kpu/kenlm.git
+ branch = 56fdb5c44fca34d5a2e07d96139c28fb163983c5
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/build.xml
----------------------------------------------------------------------
diff --git a/build.xml b/build.xml
index 7095ca2..c0c6132 100644
--- a/build.xml
+++ b/build.xml
@@ -63,7 +63,7 @@
<!-- TODO: add suggestion to use /System/Library/Frameworks/JavaVM.framework/Home/ iff on OSX -->
</target>
- <target name="kenlm" depends="check-joshua-home" description="--> Build KenLM">
+ <target name="kenlm" depends="check-joshua-home,download-submodules" description="--> Build KenLM">
<exec dir="${JOSHUA}/jni" executable="/bin/bash">
<env key="JOSHUA" path="${JOSHUA}" />
<arg line="build_kenlm.sh" />
[29/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/build.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/build.jam b/ext/kenlm/jam-files/engine/build.jam
deleted file mode 100644
index f13d950..0000000
--- a/ext/kenlm/jam-files/engine/build.jam
+++ /dev/null
@@ -1,1022 +0,0 @@
-#~ Copyright 2002-2007 Rene Rivera.
-#~ Distributed under the Boost Software License, Version 1.0.
-#~ (See accompanying file LICENSE_1_0.txt or copy at
-#~ http://www.boost.org/LICENSE_1_0.txt)
-
-# Clean env vars of any "extra" empty values.
-for local v in ARGV CC CFLAGS LIBS
-{
- local values ;
- for local x in $($(v))
- {
- if $(x) != ""
- {
- values += $(x) ;
- }
- }
- $(v) = $(values) ;
-}
-
-# Platform related specifics.
-if $(OS) = NT { rule .path { return "$(<:J=\\)" ; } ./ = "/" ; }
-else { rule .path { return "$(<:J=/)" ; } }
-
-. = "." ;
-./ ?= "" ;
-
-# Info about what we are building.
-_VERSION_ = 3 1 19 ;
-NAME = boost-jam ;
-VERSION = $(_VERSION_:J=$(.)) ;
-RELEASE = 1 ;
-LICENSE = LICENSE_1_0 ;
-
-# Generate development debug binaries?
-if --debug in $(ARGV)
-{
- debug = true ;
-}
-
-if --profile in $(ARGV)
-{
- profile = true ;
-}
-
-# Attempt to generate and/or build the grammar?
-if --grammar in $(ARGV)
-{
- grammar = true ;
-}
-
-# Do we need to add a default build type argument?
-if ! ( --release in $(ARGV) ) &&
- ! ( --debug in $(ARGV) ) &&
- ! ( --profile in $(ARGV) )
-{
- ARGV += --release ;
-}
-
-# Enable, and configure, Python hooks.
-with-python = ;
-python-location = [ MATCH --with-python=(.*) : $(ARGV) ] ;
-if $(python-location)
-{
- with-python = true ;
-}
-if $(with-python)
-{
- if $(OS) = NT
- {
- --python-include = [ .path $(python-location) include ] ;
- --python-lib = ;
- for local v in 27 26 25 24 23 22
- {
- --python-lib ?=
- [ GLOB [ .path $(python-location) libs ] : "python$(v).lib" ]
- [ GLOB $(python-location) [ .path $(python-location) libs ]
- $(Path) $(PATH) $(path) : "python$(v).dll" ]
- ;
- if ! $(--python-lib[2])
- {
- --python-lib = ;
- }
- }
- --python-lib = $(--python-lib[1]) ;
- }
- else if $(OS) = MACOSX
- {
- --python-include = [ .path $(python-location) Headers ] ;
- --python-lib = $(python-location) Python ;
- }
- else
- {
- --python-include = ;
- --python-lib = ;
- for local v in 2.7 2.6 2.5 2.4 2.3 2.2
- {
- local inc = [ GLOB [ .path $(python-location) include ] : python$(v) ] ;
- local lib = [ GLOB [ .path $(python-location) lib ] : libpython$(v)* ] ;
- if $(inc) && $(lib)
- {
- --python-include ?= $(inc) ;
- --python-lib ?= $(lib[1]:D) python$(v) ;
- }
- }
- }
-}
-
-# Boehm GC?
-if --gc in $(ARGV)
-{
- --boehm-gc = true ;
-}
-if $(--boehm-gc)
-{
- --extra-include += [ .path [ PWD ] "boehm_gc" "include" ] ;
-}
-
-# Duma?
-if --duma in $(ARGV)
-{
- --duma = true ;
-}
-if $(--duma)
-{
- --extra-include += [ .path [ PWD ] "duma" ] ;
-}
-
-# An explicit root for the toolset? (trim spaces)
-toolset-root = [ MATCH --toolset-root=(.*) : $(ARGV) ] ;
-{
- local t = [ MATCH "[ ]*(.*)" : $(toolset-root:J=" ") ] ;
- toolset-root = ;
- while $(t)
- {
- t = [ MATCH "([^ ]+)([ ]*)(.*)" : $(t) ] ;
- toolset-root += $(t[1]) ;
- if $(t[3]) { toolset-root += $(t[2]) ; }
- t = $(t[3]) ;
- }
- toolset-root = $(toolset-root:J="") ;
-}
-
-# Configure the implemented toolsets. These are minimal commands and options to
-# compile the full Jam. When adding new toolsets make sure to add them to the
-# "known" list also.
-
-rule toolset ( name command .type ? : opt.out + : opt.define * : flags * : linklibs * )
-{
- .type ?= "" ;
- tool.$(name)$(.type).cc ?= $(command) ;
- tool.$(name)$(.type).opt.out ?= $(opt.out) ;
- tool.$(name)$(.type).opt.define ?= $(opt.define) ;
- tool.$(name)$(.type).flags ?= $(flags) ;
- tool.$(name)$(.type).linklibs ?= $(linklibs) ;
- if ! $(name) in $(toolsets) { toolsets += $(name) ; }
-}
-
-rule if-os ( os + : yes-opt * : no-opt * )
- { if $(os) in $(OS) { return $(yes-opt) ; } else { return $(no-opt) ; } }
-
-rule opt ( type : yes-opt * : no-opt * )
- { if $(type) in $(ARGV) { return $(yes-opt) ; } else { return $(no-opt) ; } }
-
-## HP-UX aCC compiler
-toolset acc cc : "-o " : -D
- : -Ae
- [ opt --release : -s -O3 ]
- [ opt --debug : -g -pg ]
- -I$(--python-include) -I$(--extra-include)
- : -L$(--python-lib[1]) -l$(--python-lib[2]) ;
-## Borland C++ 5.5.x
-toolset borland bcc32 : -e -n : /D
- : -WC -w- -q "-I$(toolset-root)Include" "-L$(toolset-root)Lib"
- [ opt --release : -O2 -vi -w-inl ]
- [ opt --debug : -v -Od -vi- ]
- -I$(--python-include) -I$(--extra-include)
- : $(--python-lib[1]) ;
-## Generic Unix cc
-if ! $(CC) { CC = cc ; }
-toolset cc $(CC) : "-o " : -D
- : $(CFLAGS)
- [ opt --release : -s -O ]
- [ opt --debug : -g ]
- -I$(--python-include) -I$(--extra-include)
- : $(LIBS) -L$(--python-lib[1]) -l$(--python-lib[2]) ;
-## Comeau C/C++ 4.x
-toolset como como : "-o " : -D
- : --c
- [ opt --release : --inlining ]
- [ opt --debug : --no_inlining ]
- -I$(--python-include) -I$(--extra-include)
- : -L$(--python-lib[1]) -l$(--python-lib[2]) ;
-## Clang Linux 2.8+
-toolset clang clang : "-o " : -D
- : -Wno-unused -Wno-format
- [ opt --release : -Os ]
- [ opt --debug : -g -O0 -fno-inline ]
- [ opt --profile : -finline-functions -g ]
- -I$(--python-include) -I$(--extra-include)
- : -L$(--python-lib[1]) -l$(--python-lib[2]) ;
-## MacOSX Darwin, using GCC 2.9.x, 3.x
-toolset darwin cc : "-o " : -D
- :
- [ opt --release : -Wl,-x -O3 -finline-functions ]
- [ opt --debug : -g -O0 -fno-inline -pg ]
- [ opt --profile : -Wl,-x -O3 -finline-functions -g -pg ]
- -I$(--python-include) -I$(--extra-include)
- : -L$(--python-lib[1]) -l$(--python-lib[2]) ;
-## GCC 2.x, 3.x, 4.x
-toolset gcc gcc : "-o " : -D
- : -pedantic -fno-strict-aliasing
- [ opt --release : [ opt --symbols : -g : -s ] -O3 ]
- [ opt --debug : -g -O0 -fno-inline ]
- [ opt --profile : -O3 -g -pg ]
- -I$(--python-include) -I$(--extra-include) -Wno-long-long
- : -L$(--python-lib[1]) -l$(--python-lib[2]) ;
-## GCC 2.x, 3.x on CYGWIN but without cygwin1.dll
-toolset gcc-nocygwin gcc : "-o " : -D
- : -s -O3 -mno-cygwin
- [ opt --release : -finline-functions ]
- [ opt --debug : -s -O3 -fno-inline -pg ]
- -I$(--python-include) -I$(--extra-include)
- : -L$(--python-lib[1]) -l$(--python-lib[2]) ;
-## Intel C/C++ for Darwin
-toolset intel-darwin icc : "-o " : -D
- :
- [ opt --release : -O3 ]
- [ opt --debug : -g -O0 -p ]
- -I$(--python-include) -I$(--extra-include)
- : -L$(--python-lib[1]) -l$(--python-lib[2]) ;
-## Intel C/C++ for Linux
-toolset intel-linux icc : "-o " : -D
- :
- [ opt --release : -Xlinker -s -O3 ]
- [ opt --debug : -g -O0 -p ]
- -I$(--python-include) -I$(--extra-include)
- : -L$(--python-lib[1]) -l$(--python-lib[2]) ;
-## Intel C/C++ for Win32
-toolset intel-win32 icl : /Fe : -D
- : /nologo
- [ opt --release : /MT /O2 /Ob2 /Gy /GF /GA /GB ]
- [ opt --debug : /MTd /DEBUG /Z7 /Od /Ob0 ]
- -I$(--python-include) -I$(--extra-include)
- : kernel32.lib advapi32.lib user32.lib $(--python-lib[1]) ;
-## KCC ?
-toolset kcc KCC : "-o " : -D
- :
- [ opt --release : -s +K2 ]
- [ opt --debug : -g +K0 ]
- -I$(--python-include) -I$(--extra-include)
- : -L$(--python-lib[1]) -l$(--python-lib[2]) ;
-## Borland Kylix
-toolset kylix bc++ : -o : -D
- : -tC -q
- [ opt --release : -O2 -vi -w-inl ]
- [ opt --debug : -v -Od -vi- ]
- -I$(--python-include) -I$(--extra-include)
- : -L$(--python-lib[1]) -l$(--python-lib[2]) ;
-## Metrowerks CodeWarrior 8.x
-{
- # Even though CW can compile all files at once, it crashes if it tries in
- # the bjam case.
- local mwcc ; if $(OS) != NT { mwcc = mwc$(OSPLAT:L) ; }
- mwcc ?= mwcc ;
- toolset metrowerks $(mwcc) : "-o " : -D
- : -c -lang c -subsystem console -cwd include
- [ opt --release : -runtime ss -opt full -inline all ]
- [ opt --debug : -runtime ssd -opt none -inline off ]
- -I$(--python-include) -I$(--extra-include) ;
- toolset metrowerks $(mwcc) .link : "-o " :
- : -subsystem console -lkernel32.lib -ladvapi32.lib -luser32.lib
- [ opt --release : -runtime ss ]
- [ opt --debug : -runtime ssd ]
- : $(--python-lib[1]) ;
-}
-## MINGW GCC
-toolset mingw gcc : "-o " : -D
- :
- [ opt --release : -s -O3 -finline-functions ]
- [ opt --debug : -g -O0 -fno-inline -pg ]
- -I$(--python-include) -I$(--extra-include)
- : $(--python-lib[2]) ;
-## MIPS Pro
-toolset mipspro cc : "-o " : -D
- :
- [ opt --release : -s -O3 -g0 -INLINE:none ]
- [ opt --debug : -g -O0 -INLINE ]
- -I$(--python-include) -I$(--extra-include)
- : -L$(--python-lib[1]) -l$(--python-lib[2]) ;
-## Microsoft Visual Studio C++ 6.x
-toolset msvc cl : /Fe /Fe /Fd /Fo : -D
- : /nologo
- [ opt --release : /ML /O2 /Ob2 /Gy /GF /GA /GB ]
- [ opt --debug : /MLd /DEBUG /Z7 /Od /Ob0 ]
- -I$(--python-include) -I$(--extra-include)
- : kernel32.lib advapi32.lib user32.lib $(--python-lib[1]) ;
-## QNX 6.x GCC 3.x/2.95.3
-toolset qcc qcc : "-o " : -D
- : -Wc,-pedantic -Wc,-fno-strict-aliasing
- [ opt --release : [ opt --symbols : -g ] -O3 -Wc,-finline-functions ]
- [ opt --debug : -g -O0 -Wc,-fno-inline ]
- -I$(--python-include) -I$(--extra-include)
- : -L$(--python-lib[1]) -l$(--python-lib[2]) ;
-## Qlogic Pathscale 2.4
-toolset pathscale pathcc : "-o " : -D
- :
- [ opt --release : -s -Ofast -O3 ]
- [ opt --debug : -g ]
- -I$(--python-include) -I$(--extra-include)
- : -L$(--python-lib[1]) -l$(--python-lib[2]) ;
-## Portland Group Pgi 6.2
-toolset pgi pgcc : "-o " : -D
- :
- [ opt --release : -s -O3 ]
- [ opt --debug : -g ]
- -I$(--python-include) -I$(--extra-include)
- : -L$(--python-lib[1]) -l$(--python-lib[2]) ;
-## Sun Workshop 6 C++
-toolset sun cc : "-o " : -D
- :
- [ opt --release : -s -xO3 ]
- [ opt --debug : -g ]
- -I$(--python-include) -I$(--extra-include)
- : -L$(--python-lib[1]) -l$(--python-lib[2]) ;
-## Sun Workshop 6 C++ (old alias)
-toolset sunpro cc : "-o " : -D
- :
- [ opt --release : -s -xO3 ]
- [ opt --debug : -g ]
- -I$(--python-include) -I$(--extra-include)
- : -L$(--python-lib[1]) -l$(--python-lib[2]) ;
-## Compaq Alpha CXX
-toolset tru64cxx cc : "-o " : -D
- :
- [ opt --release : -s -O5 -inline speed ]
- [ opt --debug : -g -O0 -pg ]
- -I$(--python-include) -I$(--extra-include)
- : -L$(--python-lib[1]) -l$(--python-lib[2]) ;
-## IBM VisualAge C++
-toolset vacpp xlc : "-o " : -D
- :
- [ opt --release : -s -O3 -qstrict -qinline ]
- [ opt --debug : -g -qNOOPTimize -qnoinline -pg ]
- -I$(--python-include) -I$(--extra-include)
- : -L$(--python-lib[1]) -l$(--python-lib[2]) [ if-os AIX : -bmaxdata:0x40000000 ] ;
-## Microsoft Visual C++ .NET 7.x
-toolset vc7 cl : /Fe /Fe /Fd /Fo : -D
- : /nologo
- [ opt --release : /ML /O2 /Ob2 /Gy /GF /GA /GB ]
- [ opt --debug : /MLd /DEBUG /Z7 /Od /Ob0 ]
- -I$(--python-include) -I$(--extra-include)
- : kernel32.lib advapi32.lib user32.lib $(--python-lib[1]) ;
-## Microsoft Visual C++ 2005
-toolset vc8 cl : /Fe /Fe /Fd /Fo : -D
- : /nologo
- [ opt --release : /MT /O2 /Ob2 /Gy /GF /GA /wd4996 ]
- [ opt --debug : /MTd /DEBUG /Z7 /Od /Ob0 /wd4996 ]
- -I$(--python-include) -I$(--extra-include)
- : kernel32.lib advapi32.lib user32.lib $(--python-lib[1]) ;
-## Microsoft Visual C++ 2008
-toolset vc9 cl : /Fe /Fe /Fd /Fo : -D
- : /nologo
- [ opt --release : /MT /O2 /Ob2 /Gy /GF /GA /wd4996 ]
- [ opt --debug : /MTd /DEBUG /Z7 /Od /Ob0 /wd4996 ]
- -I$(--python-include) -I$(--extra-include)
- : kernel32.lib advapi32.lib user32.lib $(--python-lib[1]) ;
-## Microsoft Visual C++ 2010
-toolset vc10 cl : /Fe /Fe /Fd /Fo : -D
- : /nologo
- [ opt --release : /MT /O2 /Ob2 /Gy /GF /GA /wd4996 ]
- [ opt --debug : /MTd /DEBUG /Z7 /Od /Ob0 /wd4996 ]
- -I$(--python-include) -I$(--extra-include)
- : kernel32.lib advapi32.lib user32.lib $(--python-lib[1]) ;
-toolset vc11 cl : /Fe /Fe /Fd /Fo : -D
- : /nologo
- [ opt --release : /GL /MT /O2 /Ob2 /Gy /GF /GA /wd4996 ]
- [ opt --debug : /MTd /DEBUG /Z7 /Od /Ob0 /wd4996 ]
- -I$(--python-include) -I$(--extra-include)
- : kernel32.lib advapi32.lib user32.lib $(--python-lib[1]) ;
-toolset vc12 cl : /Fe /Fe /Fd /Fo : -D
- : /nologo
- [ opt --release : /GL /MT /O2 /Ob2 /Gy /GF /GA /wd4996 ]
- [ opt --debug : /MTd /DEBUG /Z7 /Od /Ob0 /wd4996 ]
- -I$(--python-include) -I$(--extra-include)
- : kernel32.lib advapi32.lib user32.lib $(--python-lib[1]) ;
-
-# First set the build commands and options according to the
-# preset toolset.
-toolset = [ MATCH --toolset=(.*) : $(ARGV) ] ;
-if ! $(toolset)
-{
- # For some reason, the following test does not catch empty toolset.
- ECHO "###" ;
- ECHO "###" No toolset specified. Please use --toolset option. ;
- ECHO "###" ;
- ECHO "###" Known toolsets are: $(toolsets:J=", ") ;
- EXIT "###" ;
-}
-if ! $(toolset) in $(toolsets)
-{
- ECHO "###" ;
- ECHO "###" Unknown toolset: $(toolset) ;
- ECHO "###" ;
- ECHO "###" Known toolsets are: $(toolsets:J=", ") ;
- EXIT "###" ;
-}
---cc = $(tool.$(toolset).cc) ;
-if $(tool.$(toolset).opt.out[2])
-{
- if $(tool.$(toolset).opt.out[1]) = $(tool.$(toolset).opt.out[2])
- {
- --out = $(tool.$(toolset).opt.out[1]) ;
- --dir = $(tool.$(toolset).opt.out[3-]) ;
- }
- else
- {
- --bin = $(tool.$(toolset).opt.out[1]) ;
- --dir = $(tool.$(toolset).opt.out[2-]) ;
- }
-}
-else
-{
- --out = $(tool.$(toolset).opt.out) ;
-}
---def = $(tool.$(toolset).opt.define) ;
---flags = $(tool.$(toolset).flags) ;
---defs = $(tool.$(toolset).defines) ;
---libs = $(tool.$(toolset).linklibs) ;
-if $(tool.$(toolset).link.cc)
-{
- --link = $(tool.$(toolset).link.cc) ;
- if $(tool.$(toolset).link.opt.out[2])
- {
- if $(tool.$(toolset).link.opt.out[1]) = $(tool.$(toolset).link.opt.out[2])
- {
- --link-out = $(tool.$(toolset).link.opt.out[1]) ;
- --link-dir = $(tool.$(toolset).link.opt.out[3-]) ;
- }
- else
- {
- --link-bin = $(tool.$(toolset).link.opt.out[1]) ;
- --link-dir = $(tool.$(toolset).link.opt.out[2-]) ;
- }
- }
- else
- {
- --link-out = $(tool.$(toolset).link.opt.out) ;
- }
- --link-def = $(tool.$(toolset).link.opt.define) ;
- --link-flags = $(tool.$(toolset).link.flags) ;
- --link-defs = $(tool.$(toolset).link.defines) ;
- --link-libs = $(tool.$(toolset).link.linklibs) ;
-}
-
-# Put executables in platform-specific subdirectory.
-locate-target = $(LOCATE_TARGET) ;
-if $(OSPLAT)
-{
- locate-target ?= bin$(.)$(OS:L)$(OSPLAT:L) ;
- platform = $(OS:L)$(OSPLAT:L) ;
-}
-else
-{
- locate-target ?= bin$(.)$(OS:L) ;
- platform = $(OS:L) ;
-}
-if $(debug)
-{
- locate-target = [ .path $(locate-target)$(.)debug ] ;
-}
-if $(profile)
-{
- locate-target = [ .path $(locate-target)$(.)profile ] ;
-}
-else
-{
- locate-target = [ .path $(locate-target) ] ;
-}
-
-if --show-locate-target in $(ARGV)
-{
- ECHO $(locate-target) ;
-}
-
-# We have some different files for UNIX, and NT.
-jam.source =
- command.c compile.c constants.c debug.c execcmd.c frames.c function.c glob.c
- hash.c hcache.c headers.c hdrmacro.c jam.c jambase.c jamgram.c lists.c
- make.c make1.c mem.c object.c option.c output.c parse.c pathsys.c regexp.c
- rules.c scan.c search.c subst.c w32_getreg.c timestamp.c variable.c
- modules.c strings.c filesys.c builtins.c class.c cwd.c native.c md5.c
- [ .path modules set.c ] [ .path modules path.c ] [ .path modules regex.c ]
- [ .path modules property-set.c ] [ .path modules sequence.c ] [ .path modules order.c ] ;
-if $(OS) = NT
-{
- jam.source += execnt.c filent.c pathnt.c ;
-}
-else
-{
- jam.source += execunix.c fileunix.c pathunix.c ;
-}
-
-# Debug assertions, or not.
-if ! $(debug) || --noassert in $(ARGV)
-{
- --defs += NDEBUG ;
-}
-
-# Enable some optional features.
---defs += OPT_HEADER_CACHE_EXT ;
---defs += OPT_GRAPH_DEBUG_EXT ;
---defs += OPT_SEMAPHORE ;
---defs += OPT_AT_FILES ;
---defs += OPT_DEBUG_PROFILE ;
-
-# Bug fixes
---defs += OPT_FIX_TARGET_VARIABLES_EXT ;
-#~ --defs += OPT_NO_EXTERNAL_VARIABLE_SPLIT ;
-
-# Improvements
---defs += OPT_IMPROVED_PATIENCE_EXT ;
-
-# Use Boehm GC memory allocator?
-if $(--boehm-gc)
-{
- --defs += OPT_BOEHM_GC ;
- if $(debug)
- {
- --defs += GC_DEBUG ;
- }
-}
-
-if $(--duma)
-{
- --defs += OPT_DUMA ;
-}
-
-if ( $(OS) = NT ) && ! NT in $(--defs)
-{
- --defs += NT ;
-}
---defs += YYSTACKSIZE=5000 ;
-
-if $(with-python)
-{
- --defs += HAVE_PYTHON ;
-}
-
-if $(debug)
-{
- --defs += BJAM_NEWSTR_NO_ALLOCATE ;
-}
-
-
-# The basic symbolic targets...
-NOTFILE all clean dist ;
-ALWAYS clean ;
-
-# Utility rules and actions...
-rule .clean
-{
- [DELETE] clean : $(<) ;
-}
-if $(OS) = NT { actions piecemeal together existing [DELETE] {
- del /F /Q "$(>)"
-} }
-if $(UNIX) = true { actions piecemeal together existing [DELETE] {
- rm -f "$(>)"
-} }
-if $(OS) = NT {
- --chmod+w = "attrib -r " ;
-}
-if $(UNIX) = true {
- --chmod+w = "chmod +w " ;
-}
-
-rule .mkdir
-{
- NOUPDATE $(<) ;
- if $(<:P) { DEPENDS $(<) : $(<:P) ; .mkdir $(<:P) ; }
- if ! $(md<$(<)>) { [MKDIR] $(<) ; md<$(<)> = - ; }
-}
-if $(OS) = NT { actions [MKDIR] {
- md "$(<)"
-} }
-if $(UNIX) = true { actions [MKDIR] {
- mkdir "$(<)"
-} }
-
-rule .exe
-{
- local exe = $(<) ;
- if $(OS) = NT || ( $(UNIX) = true && $(OS) = CYGWIN ) { exe = $(exe:S=.exe) ; }
- LOCATE on $(exe) = $(locate-target) ;
- DEPENDS all : $(exe) ;
- .mkdir $(locate-target) ;
- if $(--link)
- {
- local objs ;
- for local s in $(>)
- {
- # Translate any subdir elements into a simple file name.
- local o = [ MATCH "([^/]+)[/]?(.+)" : $(s) ] ;
- o = $(o:J=_) ;
- o = $(o:S=.o) ;
- objs += $(o) ;
- LOCATE on $(o) = $(locate-target) ;
- DEPENDS $(exe) : $(o) ;
- DEPENDS $(o) : $(s) ;
- DEPENDS $(o) : $(locate-target) ;
- [COMPILE] $(o) : $(s) ;
- .clean $(o) ;
- }
- DEPENDS $(exe) : $(objs) ;
- DEPENDS $(exe) : $(locate-target) ;
- [COMPILE.LINK] $(exe) : $(objs) ;
- .clean $(exe) ;
- }
- else
- {
- DEPENDS $(exe) : $(>) ;
- DEPENDS $(exe) : $(locate-target) ;
- [COMPILE] $(exe) : $(>) ;
- .clean $(exe) ;
- }
- return $(exe) ;
-}
-if ! $(--def[2]) { actions [COMPILE] {
- "$(--cc)" "$(--bin)$(<:D=)" "$(--dir)$(<:D)$(./)" $(--out)$(<) "$(--def)$(--defs)" "$(--flags)" "$(>)" "$(--libs)"
-} }
-else { actions [COMPILE] {
- "$(--cc)" "$(--bin)$(<:D=)" "$(--dir)$(<:D)$(./)" $(--out)$(<) "$(--def[1])$(--defs:J=$(--def[2]))$(--def[3])" "$(--flags)" "$(>)" "$(--libs)"
-} }
-
-actions [COMPILE.LINK] {
- "$(--link)" "$(--link-bin)$(<:D=)" "$(--link-dir)$(<:D)$(./)" "$(--link-out)$(<)" "$(--link-def)$(--link-defs)" "$(--link-flags)" "$(>)" "$(--link-libs)"
-}
-
-rule .link
-{
- DEPENDS all : $(<) ;
- DEPENDS $(<) : $(>) ;
- [LINK] $(<) : $(>) ;
- .clean $(<) ;
-}
-if $(OS) = NT { actions [LINK] {
- copy "$(>)" "$(<)"
-} }
-if $(UNIX) = true { actions [LINK] {
- ln -fs "$(>)" "$(<)"
-} }
-
-rule .copy
-{
- DEPENDS all : $(<) ;
- DEPENDS $(<) : $(>) ;
- [COPY] $(<) : $(>) ;
- .clean $(<) ;
-}
-
-# Will be redefined later.
-actions [COPY]
-{
-}
-
-
-rule .move
-{
- DEPENDS $(<) : $(>) ;
- [MOVE] $(<) : $(>) ;
-}
-if $(OS) = NT { actions [MOVE] {
- del /f "$(<)"
- rename "$(>)" "$(<)"
-} }
-if $(UNIX) = true { actions [MOVE] {
- mv -f "$(>)" "$(<)"
-} }
-
-# Generate the grammar tokens table, and the real yacc grammar.
-rule .yyacc
-{
- local exe = [ .exe yyacc : yyacc.c ] ;
- NOUPDATE $(exe) ;
- DEPENDS $(<) : $(exe) $(>) ;
- LEAVES $(<) ;
- yyacc.exe on $(<) = $(exe:R=$(locate-target)) ;
- [YYACC] $(<) : $(>) ;
-}
-actions [YYACC] {
- $(--chmod+w)$(<[1])
- $(--chmod+w)$(<[2])
- "$(yyacc.exe)" "$(<)" "$(>)"
-}
-if $(grammar)
-{
- .yyacc jamgram.y jamgramtab.h : jamgram.yy ;
-}
-else if $(debug)
-{
- .exe yyacc : yyacc.c ;
-}
-
-# How to build the grammar.
-if $(OS) = NT
-{
- SUFEXE = .exe ;
- # try some other likely spellings...
- PATH ?= $(Path) ;
- PATH ?= $(path) ;
-}
-SUFEXE ?= "" ;
-
-yacc ?= [ GLOB $(PATH) : yacc$(SUFEXE) ] ;
-yacc ?= [ GLOB $(PATH) : bison$(SUFEXE) ] ;
-yacc ?= [ GLOB "$(ProgramFiles:J= )\\GnuWin32\\bin"
- "C:\\Program Files\\GnuWin32\\bin" : bison$(SUFEXE) ] ;
-yacc = $(yacc[1]) ;
-switch $(yacc:D=:S=)
-{
- case bison : yacc += -d --yacc ;
- case yacc : yacc += -d ;
-}
-if $(debug) && $(yacc)
-{
- yacc += -t -v ;
-}
-yacc += $(YACCFLAGS) ;
-
-rule .yacc
-{
- DEPENDS $(<) : $(>) ;
- LEAVES $(<) ;
- [YACC] $(<) : $(>) ;
-}
-if $(OS) = NT { actions [YACC] {
- "$(yacc)" "$(>)"
- if not errorlevel 1 (
- del /f "$(<[1])"
- rename y.tab$(<[1]:S) "$(<[1])"
- del /f $(<[2])
- rename y.tab$(<[2]:S) "$(<[2])"
- ) else set _error_ =
-} }
-if $(UNIX) = true { actions [YACC] {
- if ` "$(yacc)" "$(>)" ` ; then
- mv -f y.tab$(<[1]:S) "$(<[1])"
- mv -f y.tab$(<[2]:S) "$(<[2])"
- else
- exit 1
- fi
-} }
-if $(grammar) && ! $(yacc)
-{
- EXIT Could not find the 'yacc' tool, and therefore can not build the
- grammar. ;
-}
-if $(grammar) && $(yacc)
-{
- .yacc jamgram.c jamgram.h : jamgram.y ;
-}
-
-# How to build the compiled in jambase.
-rule .mkjambase
-{
- local exe = [ .exe mkjambase : mkjambase.c ] ;
- DEPENDS $(<) : $(exe) $(>) ;
- LEAVES $(<) ;
- mkjambase.exe on $(<) = $(exe:R=$(locate-target)) ;
- [MKJAMBASE] $(<) : $(>) ;
-}
-actions [MKJAMBASE] {
- $(--chmod+w)$(<)
- $(mkjambase.exe) "$(<)" "$(>)"
-}
-if $(debug)
-{
- .mkjambase jambase.c : Jambase ;
-}
-
-# How to build Jam.
-rule .jam
-{
- $(>).exe = [ .exe $(>) : $(jam.source) ] ;
- DEPENDS all : $($(>).exe) ;
-
- # Make a copy under the old name.
- $(<).exe = $(<:S=$($(>).exe:S)) ;
- LOCATE on $($(<).exe) = $(locate-target) ;
- .copy $($(<).exe) : $($(>).exe) ;
- DEPENDS all : $($(<).exe) ;
-}
-.jam bjam : b2 ;
-
-
-# Scan sources for header dependencies.
-#
-# In order to keep things simple, we made a slight compromise here - we only
-# detect changes in headers included relative to the current folder as opposed
-# to those included from somewhere on the include path.
-rule .scan ( targets + )
-{
- HDRRULE on $(targets) = .hdr.scan ;
- HDRSCAN on $(targets) = "^[ \t]*#[ \t]*include[ \t]*\"([^\"]*)\".*$" ;
-}
-rule .hdr.scan ( target : includes * : binding )
-{
- local target-path = [ NORMALIZE_PATH $(binding:D) ] ;
- # Extra grist provides target name uniqueness when referencing same name
- # header files from different folders.
- local include-targets = <$(target-path)>$(includes) ;
- NOCARE $(include-targets) ;
- INCLUDES $(target) : $(include-targets) ;
- SEARCH on $(include-targets) = $(target-path) ;
- ISFILE $(include-targets) ;
- .scan $(include-targets) ;
-}
-.scan $(jam.source) ;
-
-
-# Distribution making from here on out. Assumes that the docs are already built
-# as HTML at ../doc/html. Otherwise they will not be included in the built
-# distribution archive.
-dist.license =
- [ GLOB . : $(LICENSE).txt ]
- ;
-dist.license = $(dist.license:D=)
- [ GLOB [ .path .. .. .. ] : $(LICENSE).txt ]
- [ GLOB [ .path .. boost ] : $(LICENSE).txt ] ;
-dist.docs =
- [ GLOB . : *.png *.css *.html ]
- ;
-dist.docs = $(dist.docs:D=)
- [ GLOB [ .path images ] : *.png ]
- [ GLOB [ .path jam ] : *.html ]
- ;
-dist.source =
- [ GLOB . : *.c *.h ]
- ;
-dist.source = $(dist.source:D=)
- $(dist.license[1])
- $(dist.docs)
- build.jam build.bat build.sh
- Jambase
- jamgram.y jamgram.yy
- [ .path modules set.c ]
- [ .path modules path.c ]
- [ .path modules regex.c ]
- [ .path modules property-set.c ]
- [ .path modules sequence.c ]
- [ .path modules order.c ]
- [ GLOB [ .path boehm_gc ] : * ]
- [ GLOB [ .path boehm_gc include ] : * ]
- [ GLOB [ .path boehm_gc include private ] : * ]
- [ GLOB [ .path boehm_gc cord ] : * ]
- [ GLOB [ .path boehm_gc Mac_files ] : * ]
- [ GLOB [ .path boehm_gc tests ] : * ]
- [ GLOB [ .path boehm_gc doc ] : * ]
- ;
-dist.bin =
- bjam
- ;
-dist.bin =
- $(dist.license[1])
- $(dist.bin:S=$(bjam.exe:S))
- ;
-
-if $(OS) = NT
-{
- zip ?= [ GLOB "$(ProgramFiles:J= )\\7-ZIP" "C:\\Program Files\\7-ZIP" : "7z.exe" ] ;
- zip ?= [ GLOB "$(ProgramFiles:J= )\\7-ZIP" "C:\\Program Files\\7-ZIP" : "7zn.exe" ] ;
- zip ?= [ GLOB $(PATH) : zip.exe ] ;
- zip ?= zip ;
- zip = $(zip[1]) ;
- switch $(zip:D=:S=)
- {
- case 7z* : zip += a -r -tzip -mx=9 ;
- case zip : zip += -9r ;
- }
- actions piecemeal [PACK] {
- "$(zip)" "$(<)" "$(>)"
- }
- actions piecemeal [ZIP] {
- "$(zip)" "$(<)" "$(>)"
- }
- actions piecemeal [COPY] {
- copy /Y "$(>)" "$(<)" >NUL:
- }
-}
-if $(UNIX) = true
-{
- tar ?= [ GLOB $(PATH) : star bsdtar tar ] ;
- tar = $(tar[1]) ;
- switch $(tar:D=:S=)
- {
- case star : tar += -c artype=pax -D -d -to-stdout ;
- case * : tar += -c -f - ;
- }
- actions [PACK] {
- "$(tar)" "$(>)" | gzip -c9 > "$(<)"
- }
- #~ actions [PACK] {
- #~ tar cf "$(<:S=.tar)" "$(>)"
- #~ }
- actions [ZIP] {
- gzip -c9 "$(>)" > "$(<)"
- }
- actions [COPY] {
- cp -Rpf "$(>)" "$(<)"
- }
-}
-
-# The single binary, compressed.
-rule .binary
-{
- local zip ;
- if $(OS) = NT { zip = $($(<).exe:S=.zip) ; }
- if $(UNIX) = true { zip = $($(<).exe:S=.tgz) ; }
- zip = $(zip:S=)-$(VERSION)-$(RELEASE)-$(platform)$(zip:S) ;
- DEPENDS $(zip) : $($(<).exe) ;
- DEPENDS dist : $(zip) ;
- #~ LOCATE on $(zip) = $(locate-target) ;
- if $(OS) = NT { [ZIP] $(zip) : $($(<).exe) ; }
- if $(UNIX) = true { [PACK] $(zip) : $($(<).exe) ; }
- .clean $(zip) ;
-}
-
-# Package some file.
-rule .package ( dst-dir : src-files + )
-{
- local dst-files ;
- local src-files-actual ;
- for local src-path in $(src-files)
- {
- if ! [ GLOB $(src-path:P) : $(src-path:B) ] || [ CHECK_IF_FILE $(src-path) ]
- {
- local src-subdir = $(src-path:D) ;
- local src-file = $(src-path) ;
- while $(src-subdir:D) { src-subdir = $(src-subdir:D) ; }
- if $(src-subdir) = ".."
- {
- src-file = $(src-file:D=) ;
- }
- dst-files += $(src-file:R=$(dst-dir)) ;
- src-files-actual += $(src-path) ;
- }
- }
-
- local pack ;
- if $(OS) = NT { pack = $(dst-dir).zip ; }
- if $(UNIX) = true { pack = $(dst-dir).tgz ; }
-
- DEPENDS dist : $(pack) ;
- DEPENDS $(pack) : $(dst-files) ;
-
- local dst-files-queue = $(dst-files) ;
- for local src-path in $(src-files-actual)
- {
- local dst-file = $(dst-files-queue[1]) ;
- dst-files-queue = $(dst-files-queue[2-]) ;
- DEPENDS $(dst-file) : $(src-path) $(dst-file:D) ;
- .mkdir $(dst-file:D) ;
-
- [COPY] $(dst-file) : $(src-path) ;
- .clean $(dst-file) ;
- }
-
- [PACK] $(pack) : $(dst-files) ;
- .clean $(pack) ;
-}
-
-# RPM distro file.
-rpm-tool = [ GLOB $(PATH) : "rpmbuild" ] ;
-rpm-tool ?= [ GLOB $(PATH) : "rpm" ] ;
-rpm-tool = $(rpm-tool[1]) ;
-rule .rpm ( name : source )
-{
- local rpm-arch ;
- switch $(OSPLAT)
- {
- case X86 : rpm-arch ?= i386 ;
- case PPC : rpm-arch ?= ppc ;
- case AXP : rpm-arch ?= alpha ;
- # no guaranty for these:
- case IA64 : rpm-arch ?= ia64 ;
- case ARM : rpm-arch ?= arm ;
- case SPARC : rpm-arch ?= sparc ;
- case * : rpm-arch ?= other ;
- }
- local target = $(name)-rpm ;
- NOTFILE $(target) ;
- DEPENDS dist : $(target) ;
- DEPENDS $(target) : $(name).$(rpm-arch).rpm $(name).src.rpm ;
- DEPENDS $(name).$(rpm-arch).rpm : $(source) ;
- DEPENDS $(name).src.rpm : $(name).$(rpm-arch).rpm ;
- docs on $(target) = $(dist.docs:J=" ") ;
- arch on $(target) = $(rpm-arch) ;
- if $(rpm-arch) = ppc { target-opt on $(target) = --target= ; }
- else { target-opt on $(target) = "--target " ; }
- [RPM] $(target) : $(source) ;
- .clean $(name).$(rpm-arch).rpm $(name).src.rpm ;
-}
-actions [RPM] {
- set -e
- export BOOST_JAM_TOOLSET="$(toolset)"
- $(rpm-tool) -ta $(target-opt)$(arch) $(>) | tee rpm.out
- cp `grep -e '^Wrote:' rpm.out | sed 's/^Wrote: //'` .
- rm -f rpm.out
-}
-
-# The distribution targets. Do not bother with them unless this is a
-# distribution build.
-if dist in $(ARGV)
-{
- #~ .binary bjam ;
- .package $(NAME)-$(VERSION) : $(dist.source) ;
- .package $(NAME)-$(VERSION)-$(RELEASE)-$(platform) : $(dist.bin) ;
- if $(rpm-tool)
- {
- #~ .rpm $(NAME)-$(VERSION)-$(RELEASE) : $(NAME)-$(VERSION).tgz ;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/build.sh
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/build.sh b/ext/kenlm/jam-files/engine/build.sh
deleted file mode 100755
index 470ea3c..0000000
--- a/ext/kenlm/jam-files/engine/build.sh
+++ /dev/null
@@ -1,303 +0,0 @@
-#!/bin/sh
-
-#~ Copyright 2002-2005 Rene Rivera.
-#~ Distributed under the Boost Software License, Version 1.0.
-#~ (See accompanying file LICENSE_1_0.txt or copy at
-#~ http://www.boost.org/LICENSE_1_0.txt)
-
-# Reset the toolset.
-BOOST_JAM_TOOLSET=
-
-# Run a command, and echo before doing so. Also checks the exit status and quits
-# if there was an error.
-echo_run ()
-{
- echo "$@"
- $@
- r=$?
- if test $r -ne 0 ; then
- exit $r
- fi
-}
-
-# Print an error message, and exit with a status of 1.
-error_exit ()
-{
- echo "###"
- echo "###" "$@"
- echo "###"
- echo "### You can specify the toolset as the argument, i.e.:"
- echo "### ./build.sh gcc"
- echo "###"
- echo "### Toolsets supported by this script are:"
- echo "### acc, como, darwin, gcc, intel-darwin, intel-linux, kcc, kylix,"
- echo "### mipspro, mingw(msys), pathscale, pgi, qcc, sun, sunpro, tru64cxx, vacpp"
- echo "###"
- echo "### A special toolset; cc, is available which is used as a fallback"
- echo "### when a more specific toolset is not found and the cc command is"
- echo "### detected. The 'cc' toolset will use the CC, CFLAGS, and LIBS"
- echo "### environment variables, if present."
- echo "###"
- exit 1
-}
-
-# Check that a command is in the PATH.
-test_path ()
-{
- if `command -v command 1>/dev/null 2>/dev/null`; then
- command -v $1 1>/dev/null 2>/dev/null
- else
- hash $1 1>/dev/null 2>/dev/null
- fi
-}
-
-# Check that the OS name, as returned by "uname", is as given.
-test_uname ()
-{
- if test_path uname; then
- test `uname` = $*
- fi
-}
-
-# Try and guess the toolset to bootstrap the build with...
-Guess_Toolset ()
-{
- if test -r /mingw/bin/gcc ; then
- BOOST_JAM_TOOLSET=mingw
- BOOST_JAM_TOOLSET_ROOT=/mingw/
- elif test_uname Darwin ; then BOOST_JAM_TOOLSET=darwin
- elif test_uname IRIX ; then BOOST_JAM_TOOLSET=mipspro
- elif test_uname IRIX64 ; then BOOST_JAM_TOOLSET=mipspro
- elif test_uname OSF1 ; then BOOST_JAM_TOOLSET=tru64cxx
- elif test_uname QNX && test_path qcc ; then BOOST_JAM_TOOLSET=qcc
- elif test_path gcc ; then BOOST_JAM_TOOLSET=gcc
- elif test_path icc ; then BOOST_JAM_TOOLSET=intel-linux
- elif test -r /opt/intel/cc/9.0/bin/iccvars.sh ; then
- BOOST_JAM_TOOLSET=intel-linux
- BOOST_JAM_TOOLSET_ROOT=/opt/intel/cc/9.0
- elif test -r /opt/intel_cc_80/bin/iccvars.sh ; then
- BOOST_JAM_TOOLSET=intel-linux
- BOOST_JAM_TOOLSET_ROOT=/opt/intel_cc_80
- elif test -r /opt/intel/compiler70/ia32/bin/iccvars.sh ; then
- BOOST_JAM_TOOLSET=intel-linux
- BOOST_JAM_TOOLSET_ROOT=/opt/intel/compiler70/ia32/
- elif test -r /opt/intel/compiler60/ia32/bin/iccvars.sh ; then
- BOOST_JAM_TOOLSET=intel-linux
- BOOST_JAM_TOOLSET_ROOT=/opt/intel/compiler60/ia32/
- elif test -r /opt/intel/compiler50/ia32/bin/iccvars.sh ; then
- BOOST_JAM_TOOLSET=intel-linux
- BOOST_JAM_TOOLSET_ROOT=/opt/intel/compiler50/ia32/
- elif test_path pgcc ; then BOOST_JAM_TOOLSET=pgi
- elif test_path pathcc ; then BOOST_JAM_TOOLSET=pathscale
- elif test_path xlc ; then BOOST_JAM_TOOLSET=vacpp
- elif test_path como ; then BOOST_JAM_TOOLSET=como
- elif test_path KCC ; then BOOST_JAM_TOOLSET=kcc
- elif test_path bc++ ; then BOOST_JAM_TOOLSET=kylix
- elif test_path aCC ; then BOOST_JAM_TOOLSET=acc
- elif test_uname HP-UX ; then BOOST_JAM_TOOLSET=acc
- elif test -r /opt/SUNWspro/bin/cc ; then
- BOOST_JAM_TOOLSET=sunpro
- BOOST_JAM_TOOLSET_ROOT=/opt/SUNWspro/
- # Test for "cc" as the default fallback.
- elif test_path $CC ; then BOOST_JAM_TOOLSET=cc
- elif test_path cc ; then
- BOOST_JAM_TOOLSET=cc
- CC=cc
- fi
- if test "$BOOST_JAM_TOOLSET" = "" ; then
- error_exit "Could not find a suitable toolset."
- fi
-}
-
-# The one option we support in the invocation
-# is the name of the toolset to force building
-# with.
-case "$1" in
- --guess-toolset) Guess_Toolset ; echo "$BOOST_JAM_TOOLSET" ; exit 1 ;;
- -*) Guess_Toolset ;;
- ?*) BOOST_JAM_TOOLSET=$1 ; shift ;;
- *) Guess_Toolset ;;
-esac
-BOOST_JAM_OPT_JAM="-o bootstrap/jam0"
-BOOST_JAM_OPT_MKJAMBASE="-o bootstrap/mkjambase0"
-BOOST_JAM_OPT_YYACC="-o bootstrap/yyacc0"
-case $BOOST_JAM_TOOLSET in
- mingw)
- if test -r ${BOOST_JAM_TOOLSET_ROOT}bin/gcc ; then
- export PATH=${BOOST_JAM_TOOLSET_ROOT}bin:$PATH
- fi
- BOOST_JAM_CC="gcc -DNT"
- ;;
-
- gcc)
- BOOST_JAM_CC=gcc
- ;;
-
- darwin)
- BOOST_JAM_CC=cc
- ;;
-
- intel-darwin)
- BOOST_JAM_CC=icc
- ;;
-
- intel-linux)
- if test -r /opt/intel/cc/9.0/bin/iccvars.sh ; then
- BOOST_JAM_TOOLSET_ROOT=/opt/intel/cc/9.0/
- elif test -r /opt/intel_cc_80/bin/iccvars.sh ; then
- BOOST_JAM_TOOLSET_ROOT=/opt/intel_cc_80/
- elif test -r /opt/intel/compiler70/ia32/bin/iccvars.sh ; then
- BOOST_JAM_TOOLSET_ROOT=/opt/intel/compiler70/ia32/
- elif test -r /opt/intel/compiler60/ia32/bin/iccvars.sh ; then
- BOOST_JAM_TOOLSET_ROOT=/opt/intel/compiler60/ia32/
- elif test -r /opt/intel/compiler50/ia32/bin/iccvars.sh ; then
- BOOST_JAM_TOOLSET_ROOT=/opt/intel/compiler50/ia32/
- fi
- if test -r ${BOOST_JAM_TOOLSET_ROOT}bin/iccvars.sh ; then
- # iccvars does not change LD_RUN_PATH. We adjust LD_RUN_PATH here in
- # order not to have to rely on ld.so.conf knowing the icc library
- # directory. We do this before running iccvars.sh in order to allow a
- # user to add modifications to LD_RUN_PATH in iccvars.sh.
- if test -z "${LD_RUN_PATH}"; then
- LD_RUN_PATH="${BOOST_JAM_TOOLSET_ROOT}lib"
- else
- LD_RUN_PATH="${BOOST_JAM_TOOLSET_ROOT}lib:${LD_RUN_PATH}"
- fi
- export LD_RUN_PATH
- . ${BOOST_JAM_TOOLSET_ROOT}bin/iccvars.sh
- fi
- BOOST_JAM_CC=icc
- ;;
-
- vacpp)
- BOOST_JAM_CC=xlc
- ;;
-
- como)
- BOOST_JAM_CC="como --c"
- ;;
-
- kcc)
- BOOST_JAM_CC=KCC
- ;;
-
- kylix)
- BOOST_JAM_CC=bc++
- ;;
-
- mipspro)
- BOOST_JAM_CC=cc
- ;;
-
- pathscale)
- BOOST_JAM_CC=pathcc
- ;;
-
- pgi)
- BOOST_JAM_CC=pgcc
- ;;
-
- sun*)
- if test -z "${BOOST_JAM_TOOLSET_ROOT}" -a -r /opt/SUNWspro/bin/cc ; then
- BOOST_JAM_TOOLSET_ROOT=/opt/SUNWspro/
- fi
- if test -r "${BOOST_JAM_TOOLSET_ROOT}bin/cc" ; then
- PATH=${BOOST_JAM_TOOLSET_ROOT}bin:${PATH}
- export PATH
- fi
- BOOST_JAM_CC=cc
- ;;
-
- clang*)
- BOOST_JAM_CC="clang -Wno-unused -Wno-format"
- BOOST_JAM_TOOLSET=clang
- ;;
-
- tru64cxx)
- BOOST_JAM_CC=cc
- ;;
-
- acc)
- BOOST_JAM_CC="cc -Ae"
- ;;
-
- cc)
- if test -z "$CC" ; then CC=cc ; fi
- BOOST_JAM_CC=$CC
- BOOST_JAM_OPT_JAM="$BOOST_JAM_OPT_JAM $CFLAGS $LIBS"
- BOOST_JAM_OPT_MKJAMBASE="$BOOST_JAM_OPT_MKJAMBASE $CFLAGS $LIBS"
- BOOST_JAM_OPT_YYACC="$BOOST_JAM_OPT_YYACC $CFLAGS $LIBS"
- ;;
-
- qcc)
- BOOST_JAM_CC=qcc
- ;;
-
- *)
- error_exit "Unknown toolset: $BOOST_JAM_TOOLSET"
- ;;
-esac
-
-echo "###"
-echo "### Using '$BOOST_JAM_TOOLSET' toolset."
-echo "###"
-
-YYACC_SOURCES="yyacc.c"
-MKJAMBASE_SOURCES="mkjambase.c"
-BJAM_SOURCES="\
- command.c compile.c constants.c debug.c execcmd.c frames.c function.c glob.c\
- hash.c hdrmacro.c headers.c jam.c jambase.c jamgram.c lists.c make.c make1.c\
- object.c option.c output.c parse.c pathsys.c regexp.c rules.c\
- scan.c search.c subst.c timestamp.c variable.c modules.c strings.c filesys.c\
- builtins.c class.c cwd.c native.c md5.c w32_getreg.c modules/set.c\
- modules/path.c modules/regex.c modules/property-set.c modules/sequence.c\
- modules/order.c"
-case $BOOST_JAM_TOOLSET in
- mingw)
- BJAM_SOURCES="${BJAM_SOURCES} execnt.c filent.c pathnt.c"
- ;;
-
- *)
- BJAM_SOURCES="${BJAM_SOURCES} execunix.c fileunix.c pathunix.c"
- ;;
-esac
-
-BJAM_UPDATE=
-if test "$1" = "--update" -o "$2" = "--update" -o "$3" = "--update" -o "$4" = "--update" ; then
- BJAM_UPDATE="update"
-fi
-if test "${BJAM_UPDATE}" = "update" -a ! -x "./bootstrap/jam0" ; then
- BJAM_UPDATE=
-fi
-
-if test "${BJAM_UPDATE}" != "update" ; then
- echo_run rm -rf bootstrap
- echo_run mkdir bootstrap
- if test ! -r jamgram.y -o ! -r jamgramtab.h ; then
- echo_run ${BOOST_JAM_CC} ${BOOST_JAM_OPT_YYACC} ${YYACC_SOURCES}
- if test -x "./bootstrap/yyacc0" ; then
- echo_run ./bootstrap/yyacc0 jamgram.y jamgramtab.h jamgram.yy
- fi
- fi
- if test ! -r jamgram.c -o ! -r jamgram.h ; then
- if test_path yacc ; then YACC="yacc -d"
- elif test_path bison ; then YACC="bison -y -d --yacc"
- fi
- echo_run $YACC jamgram.y
- mv -f y.tab.c jamgram.c
- mv -f y.tab.h jamgram.h
- fi
- if test ! -r jambase.c ; then
- echo_run ${BOOST_JAM_CC} ${BOOST_JAM_OPT_MKJAMBASE} ${MKJAMBASE_SOURCES}
- if test -x "./bootstrap/mkjambase0" ; then
- echo_run ./bootstrap/mkjambase0 jambase.c Jambase
- fi
- fi
- echo_run ${BOOST_JAM_CC} ${BOOST_JAM_OPT_JAM} ${BJAM_SOURCES}
-fi
-if test -x "./bootstrap/jam0" ; then
- if test "${BJAM_UPDATE}" != "update" ; then
- echo_run ./bootstrap/jam0 -f build.jam --toolset=$BOOST_JAM_TOOLSET "--toolset-root=$BOOST_JAM_TOOLSET_ROOT" "$@" clean
- fi
- echo_run ./bootstrap/jam0 -f build.jam --toolset=$BOOST_JAM_TOOLSET "--toolset-root=$BOOST_JAM_TOOLSET_ROOT" "$@"
-fi
[25/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/fileunix.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/fileunix.c b/ext/kenlm/jam-files/engine/fileunix.c
deleted file mode 100644
index bff3a42..0000000
--- a/ext/kenlm/jam-files/engine/fileunix.c
+++ /dev/null
@@ -1,464 +0,0 @@
-/*
- * Copyright 1993-2002 Christopher Seiwald and Perforce Software, Inc.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Copyright 2005 Rene Rivera.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * fileunix.c - manipulate file names and scan directories on UNIX/AmigaOS
- *
- * External routines:
- * file_archscan() - scan an archive for files
- * file_mkdir() - create a directory
- * file_supported_fmt_resolution() - file modification timestamp resolution
- *
- * External routines called only via routines in filesys.c:
- * file_collect_dir_content_() - collects directory content information
- * file_dirscan_() - OS specific file_dirscan() implementation
- * file_query_() - query information about a path from the OS
- */
-
-#include "jam.h"
-#ifdef USE_FILEUNIX
-#include "filesys.h"
-
-#include "object.h"
-#include "pathsys.h"
-#include "strings.h"
-
-#include <assert.h>
-#include <stdio.h>
-#include <sys/stat.h> /* needed for mkdir() */
-
-#if defined( sun ) || defined( __sun ) || defined( linux )
-# include <unistd.h> /* needed for read and close prototype */
-#endif
-
-#if defined( OS_SEQUENT ) || \
- defined( OS_DGUX ) || \
- defined( OS_SCO ) || \
- defined( OS_ISC )
-# define PORTAR 1
-#endif
-
-#if defined( OS_RHAPSODY ) || defined( OS_MACOSX ) || defined( OS_NEXT )
-# include <sys/dir.h>
-# include <unistd.h> /* need unistd for rhapsody's proper lseek */
-# define STRUCT_DIRENT struct direct
-#else
-# include <dirent.h>
-# define STRUCT_DIRENT struct dirent
-#endif
-
-#ifdef OS_COHERENT
-# include <arcoff.h>
-# define HAVE_AR
-#endif
-
-#if defined( OS_MVS ) || defined( OS_INTERIX )
-#define ARMAG "!<arch>\n"
-#define SARMAG 8
-#define ARFMAG "`\n"
-#define HAVE_AR
-
-struct ar_hdr /* archive file member header - printable ascii */
-{
- char ar_name[ 16 ]; /* file member name - `/' terminated */
- char ar_date[ 12 ]; /* file member date - decimal */
- char ar_uid[ 6 ]; /* file member user id - decimal */
- char ar_gid[ 6 ]; /* file member group id - decimal */
- char ar_mode[ 8 ]; /* file member mode - octal */
- char ar_size[ 10 ]; /* file member size - decimal */
- char ar_fmag[ 2 ]; /* ARFMAG - string to end header */
-};
-#endif
-
-#if defined( OS_QNX ) || defined( OS_BEOS ) || defined( OS_MPEIX )
-# define NO_AR
-# define HAVE_AR
-#endif
-
-#ifndef HAVE_AR
-# ifdef OS_AIX
-/* Define these for AIX to get the definitions for both small and big archive
- * file format variants.
- */
-# define __AR_SMALL__
-# define __AR_BIG__
-# endif
-# include <ar.h>
-#endif
-
-
-/*
- * file_collect_dir_content_() - collects directory content information
- */
-
-int file_collect_dir_content_( file_info_t * const d )
-{
- LIST * files = L0;
- PATHNAME f;
- DIR * dd;
- STRUCT_DIRENT * dirent;
- string path[ 1 ];
- char const * dirstr;
-
- assert( d );
- assert( d->is_dir );
- assert( list_empty( d->files ) );
-
- dirstr = object_str( d->name );
-
- memset( (char *)&f, '\0', sizeof( f ) );
- f.f_dir.ptr = dirstr;
- f.f_dir.len = strlen( dirstr );
-
- if ( !*dirstr ) dirstr = ".";
-
- if ( !( dd = opendir( dirstr ) ) )
- return -1;
-
- string_new( path );
- while ( ( dirent = readdir( dd ) ) )
- {
- OBJECT * name;
- f.f_base.ptr = dirent->d_name
- #ifdef old_sinix
- - 2 /* Broken structure definition on sinix. */
- #endif
- ;
- f.f_base.len = strlen( f.f_base.ptr );
-
- string_truncate( path, 0 );
- path_build( &f, path );
- name = object_new( path->value );
- /* Immediately stat the file to preserve invariants. */
- if ( file_query( name ) )
- files = list_push_back( files, name );
- else
- object_free( name );
- }
- string_free( path );
-
- closedir( dd );
-
- d->files = files;
- return 0;
-}
-
-
-/*
- * file_dirscan_() - OS specific file_dirscan() implementation
- */
-
-void file_dirscan_( file_info_t * const d, scanback func, void * closure )
-{
- assert( d );
- assert( d->is_dir );
-
- /* Special case / : enter it */
- if ( !strcmp( object_str( d->name ), "/" ) )
- (*func)( closure, d->name, 1 /* stat()'ed */, &d->time );
-}
-
-
-/*
- * file_mkdir() - create a directory
- */
-
-int file_mkdir( char const * const path )
-{
- /* Explicit cast to remove const modifiers and avoid related compiler
- * warnings displayed when using the intel compiler.
- */
- return mkdir( (char *)path, 0777 );
-}
-
-
-/*
- * file_query_() - query information about a path from the OS
- */
-
-void file_query_( file_info_t * const info )
-{
- file_query_posix_( info );
-}
-
-
-/*
- * file_supported_fmt_resolution() - file modification timestamp resolution
- *
- * Returns the minimum file modification timestamp resolution supported by this
- * Boost Jam implementation. File modification timestamp changes of less than
- * the returned value might not be recognized.
- *
- * Does not take into consideration any OS or file system related restrictions.
- *
- * Return value 0 indicates that any value supported by the OS is also supported
- * here.
- */
-
-void file_supported_fmt_resolution( timestamp * const t )
-{
- /* The current implementation does not support file modification timestamp
- * resolution of less than one second.
- */
- timestamp_init( t, 1, 0 );
-}
-
-
-/*
- * file_archscan() - scan an archive for files
- */
-
-#ifndef AIAMAG /* God-fearing UNIX */
-
-#define SARFMAG 2
-#define SARHDR sizeof( struct ar_hdr )
-
-void file_archscan( char const * archive, scanback func, void * closure )
-{
-#ifndef NO_AR
- struct ar_hdr ar_hdr;
- char * string_table = 0;
- char buf[ MAXJPATH ];
- long offset;
- int fd;
-
- if ( ( fd = open( archive, O_RDONLY, 0 ) ) < 0 )
- return;
-
- if ( read( fd, buf, SARMAG ) != SARMAG ||
- strncmp( ARMAG, buf, SARMAG ) )
- {
- close( fd );
- return;
- }
-
- offset = SARMAG;
-
- if ( DEBUG_BINDSCAN )
- printf( "scan archive %s\n", archive );
-
- while ( ( read( fd, &ar_hdr, SARHDR ) == SARHDR ) &&
- !( memcmp( ar_hdr.ar_fmag, ARFMAG, SARFMAG )
-#ifdef ARFZMAG
- /* OSF also has a compressed format */
- && memcmp( ar_hdr.ar_fmag, ARFZMAG, SARFMAG )
-#endif
- ) )
- {
- char lar_name_[ 257 ];
- char * lar_name = lar_name_ + 1;
- long lar_date;
- long lar_size;
- long lar_offset;
- char * c;
- char * src;
- char * dest;
-
- strncpy( lar_name, ar_hdr.ar_name, sizeof( ar_hdr.ar_name ) );
-
- sscanf( ar_hdr.ar_date, "%ld", &lar_date );
- sscanf( ar_hdr.ar_size, "%ld", &lar_size );
-
- if ( ar_hdr.ar_name[ 0 ] == '/' )
- {
- if ( ar_hdr.ar_name[ 1 ] == '/' )
- {
- /* This is the "string table" entry of the symbol table, holding
- * filename strings longer than 15 characters, i.e. those that
- * do not fit into ar_name.
- */
- string_table = (char *)BJAM_MALLOC_ATOMIC( lar_size );
- lseek( fd, offset + SARHDR, 0 );
- if ( read( fd, string_table, lar_size ) != lar_size )
- printf("error reading string table\n");
- }
- else if ( string_table && ar_hdr.ar_name[ 1 ] != ' ' )
- {
- /* Long filenames are recognized by "/nnnn" where nnnn is the
- * offset of the string in the string table represented in ASCII
- * decimals.
- */
- dest = lar_name;
- lar_offset = atoi( lar_name + 1 );
- src = &string_table[ lar_offset ];
- while ( *src != '/' )
- *dest++ = *src++;
- *dest = '/';
- }
- }
-
- c = lar_name - 1;
- while ( ( *++c != ' ' ) && ( *c != '/' ) );
- *c = '\0';
-
- if ( DEBUG_BINDSCAN )
- printf( "archive name %s found\n", lar_name );
-
- sprintf( buf, "%s(%s)", archive, lar_name );
-
- {
- OBJECT * const member = object_new( buf );
- timestamp time;
- timestamp_init( &time, (time_t)lar_date, 0 );
- (*func)( closure, member, 1 /* time valid */, &time );
- object_free( member );
- }
-
- offset += SARHDR + ( ( lar_size + 1 ) & ~1 );
- lseek( fd, offset, 0 );
- }
-
- if ( string_table )
- BJAM_FREE( string_table );
-
- close( fd );
-#endif /* NO_AR */
-}
-
-#else /* AIAMAG - RS6000 AIX */
-
-static void file_archscan_small( int fd, char const * archive, scanback func,
- void * closure )
-{
- struct fl_hdr fl_hdr;
-
- struct {
- struct ar_hdr hdr;
- char pad[ 256 ];
- } ar_hdr ;
-
- char buf[ MAXJPATH ];
- long offset;
-
- if ( read( fd, (char *)&fl_hdr, FL_HSZ ) != FL_HSZ )
- return;
-
- sscanf( fl_hdr.fl_fstmoff, "%ld", &offset );
-
- if ( DEBUG_BINDSCAN )
- printf( "scan archive %s\n", archive );
-
- while ( offset > 0 && lseek( fd, offset, 0 ) >= 0 &&
- read( fd, &ar_hdr, sizeof( ar_hdr ) ) >= (int)sizeof( ar_hdr.hdr ) )
- {
- long lar_date;
- int lar_namlen;
-
- sscanf( ar_hdr.hdr.ar_namlen, "%d" , &lar_namlen );
- sscanf( ar_hdr.hdr.ar_date , "%ld", &lar_date );
- sscanf( ar_hdr.hdr.ar_nxtmem, "%ld", &offset );
-
- if ( !lar_namlen )
- continue;
-
- ar_hdr.hdr._ar_name.ar_name[ lar_namlen ] = '\0';
-
- sprintf( buf, "%s(%s)", archive, ar_hdr.hdr._ar_name.ar_name );
-
- {
- OBJECT * const member = object_new( buf );
- timestamp time;
- timestamp_init( &time, (time_t)lar_date, 0 );
- (*func)( closure, member, 1 /* time valid */, &time );
- object_free( member );
- }
- }
-}
-
-/* Check for OS versions supporting the big variant. */
-#ifdef AR_HSZ_BIG
-
-static void file_archscan_big( int fd, char const * archive, scanback func,
- void * closure )
-{
- struct fl_hdr_big fl_hdr;
-
- struct {
- struct ar_hdr_big hdr;
- char pad[ 256 ];
- } ar_hdr ;
-
- char buf[ MAXJPATH ];
- long long offset;
-
- if ( read( fd, (char *)&fl_hdr, FL_HSZ_BIG ) != FL_HSZ_BIG )
- return;
-
- sscanf( fl_hdr.fl_fstmoff, "%lld", &offset );
-
- if ( DEBUG_BINDSCAN )
- printf( "scan archive %s\n", archive );
-
- while ( offset > 0 && lseek( fd, offset, 0 ) >= 0 &&
- read( fd, &ar_hdr, sizeof( ar_hdr ) ) >= sizeof( ar_hdr.hdr ) )
- {
- long lar_date;
- int lar_namlen;
-
- sscanf( ar_hdr.hdr.ar_namlen, "%d" , &lar_namlen );
- sscanf( ar_hdr.hdr.ar_date , "%ld" , &lar_date );
- sscanf( ar_hdr.hdr.ar_nxtmem, "%lld", &offset );
-
- if ( !lar_namlen )
- continue;
-
- ar_hdr.hdr._ar_name.ar_name[ lar_namlen ] = '\0';
-
- sprintf( buf, "%s(%s)", archive, ar_hdr.hdr._ar_name.ar_name );
-
- {
- OBJECT * const member = object_new( buf );
- timestamp time;
- timestamp_init( &time, (time_t)lar_date, 0 );
- (*func)( closure, member, 1 /* time valid */, &time );
- object_free( member );
- }
- }
-}
-
-#endif /* AR_HSZ_BIG */
-
-void file_archscan( char const * archive, scanback func, void * closure )
-{
- int fd;
- char fl_magic[ SAIAMAG ];
-
- if ( ( fd = open( archive, O_RDONLY, 0 ) ) < 0 )
- return;
-
- if ( read( fd, fl_magic, SAIAMAG ) != SAIAMAG ||
- lseek( fd, 0, SEEK_SET ) == -1 )
- {
- close( fd );
- return;
- }
-
- if ( !strncmp( AIAMAG, fl_magic, SAIAMAG ) )
- {
- /* read small variant */
- file_archscan_small( fd, archive, func, closure );
- }
-#ifdef AR_HSZ_BIG
- else if ( !strncmp( AIAMAGBIG, fl_magic, SAIAMAG ) )
- {
- /* read big variant */
- file_archscan_big( fd, archive, func, closure );
- }
-#endif
-
- close( fd );
-}
-
-#endif /* AIAMAG - RS6000 AIX */
-
-#endif /* USE_FILEUNIX */
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/frames.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/frames.c b/ext/kenlm/jam-files/engine/frames.c
deleted file mode 100644
index 0491c5c..0000000
--- a/ext/kenlm/jam-files/engine/frames.c
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#include "jam.h"
-#include "frames.h"
-
-
-FRAME * frame_before_python_call;
-
-
-void frame_init( FRAME * frame )
-{
- frame->prev = 0;
- frame->prev_user = 0;
- lol_init( frame->args );
- frame->module = root_module();
- frame->rulename = "module scope";
- frame->file = 0;
- frame->line = -1;
-}
-
-
-void frame_free( FRAME * frame )
-{
- lol_free( frame->args );
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/frames.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/frames.h b/ext/kenlm/jam-files/engine/frames.h
deleted file mode 100644
index 2e99f17..0000000
--- a/ext/kenlm/jam-files/engine/frames.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#ifndef FRAMES_DWA20011021_H
-#define FRAMES_DWA20011021_H
-
-#include "lists.h"
-#include "modules.h"
-#include "object.h"
-
-
-typedef struct frame FRAME;
-
-struct frame
-{
- FRAME * prev;
- FRAME * prev_user; /* The nearest enclosing frame for which
- module->user_module is true. */
- LOL args[ 1 ];
- module_t * module;
- OBJECT * file;
- int line;
- char const * rulename;
-};
-
-
-/* When a call into Python is in progress, this variable points to the bjam
- * frame that was current at the moment of the call. When the call completes,
- * the variable is not defined. Furthermore, if Jam calls Python which calls Jam
- * and so on, this variable only keeps the most recent Jam frame.
- */
-extern FRAME * frame_before_python_call;
-
-
-void frame_init( FRAME * );
-void frame_free( FRAME * );
-
-#endif
[28/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/builtins.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/builtins.c b/ext/kenlm/jam-files/engine/builtins.c
deleted file mode 100644
index e4130bb..0000000
--- a/ext/kenlm/jam-files/engine/builtins.c
+++ /dev/null
@@ -1,2354 +0,0 @@
-/*
- * Copyright 1993-2002 Christopher Seiwald and Perforce Software, Inc.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-#include "jam.h"
-#include "builtins.h"
-
-#include "compile.h"
-#include "constants.h"
-#include "cwd.h"
-#include "filesys.h"
-#include "frames.h"
-#include "hash.h"
-#include "hdrmacro.h"
-#include "lists.h"
-#include "make.h"
-#include "md5.h"
-#include "native.h"
-#include "object.h"
-#include "parse.h"
-#include "pathsys.h"
-#include "rules.h"
-#include "strings.h"
-#include "subst.h"
-#include "timestamp.h"
-#include "variable.h"
-
-#include <ctype.h>
-
-#if defined(USE_EXECUNIX)
-# include <sys/types.h>
-# include <sys/wait.h>
-#else
-/*
- * NT does not have wait() and associated macros and uses the system() return
- * value instead. Status code group are documented at:
- * http://msdn.microsoft.com/en-gb/library/ff565436.aspx
- */
-# define WIFEXITED(w) (((w) & 0XFFFFFF00) == 0)
-# define WEXITSTATUS(w)(w)
-#endif
-
-/*
- * builtins.c - builtin jam rules
- *
- * External routines:
- * load_builtins() - define builtin rules
- * unknown_rule() - reports an unknown rule occurrence to the
- * user and exits
- *
- * Internal routines:
- * append_if_exists() - if file exists, append it to the list
- * builtin_calc() - CALC rule
- * builtin_delete_module() - DELETE_MODULE ( MODULE ? )
- * builtin_depends() - DEPENDS/INCLUDES rule
- * builtin_echo() - ECHO rule
- * builtin_exit() - EXIT rule
- * builtin_export() - EXPORT ( MODULE ? : RULES * )
- * builtin_flags() - NOCARE, NOTFILE, TEMPORARY rule
- * builtin_glob() - GLOB rule
- * builtin_glob_recursive() - ???
- * builtin_hdrmacro() - ???
- * builtin_import() - IMPORT rule
- * builtin_match() - MATCH rule, regexp matching
- * builtin_rebuilds() - REBUILDS rule
- * builtin_rulenames() - RULENAMES ( MODULE ? )
- * builtin_split_by_characters() - splits the given string into tokens
- * builtin_varnames() - VARNAMES ( MODULE ? )
- * get_source_line() - get a frame's file and line number
- * information
- */
-
-
-/*
- * compile_builtin() - define builtin rules
- */
-
-#define P0 (PARSE *)0
-#define C0 (OBJECT *)0
-
-#if defined( OS_NT ) || defined( OS_CYGWIN )
- LIST * builtin_system_registry ( FRAME *, int );
- LIST * builtin_system_registry_names( FRAME *, int );
-#endif
-
-int glob( char const * s, char const * c );
-
-void backtrace ( FRAME * );
-void backtrace_line ( FRAME * );
-void print_source_line( FRAME * );
-
-
-RULE * bind_builtin( char const * name_, LIST * (* f)( FRAME *, int flags ),
- int flags, char const * * args )
-{
- FUNCTION * func;
- RULE * result;
- OBJECT * name = object_new( name_ );
-
- func = function_builtin( f, flags, args );
-
- result = new_rule_body( root_module(), name, func, 1 );
-
- function_free( func );
-
- object_free( name );
-
- return result;
-}
-
-
-RULE * duplicate_rule( char const * name_, RULE * other )
-{
- OBJECT * name = object_new( name_ );
- RULE * result = import_rule( other, root_module(), name );
- object_free( name );
- return result;
-}
-
-
-/*
- * load_builtins() - define builtin rules
- */
-
-void load_builtins()
-{
- duplicate_rule( "Always",
- bind_builtin( "ALWAYS",
- builtin_flags, T_FLAG_TOUCHED, 0 ) );
-
- duplicate_rule( "Depends",
- bind_builtin( "DEPENDS",
- builtin_depends, 0, 0 ) );
-
- duplicate_rule( "echo",
- duplicate_rule( "Echo",
- bind_builtin( "ECHO",
- builtin_echo, 0, 0 ) ) );
-
- {
- char const * args[] = { "message", "*", ":", "result-value", "?", 0 };
- duplicate_rule( "exit",
- duplicate_rule( "Exit",
- bind_builtin( "EXIT",
- builtin_exit, 0, args ) ) );
- }
-
- {
- char const * args[] = { "directories", "*", ":", "patterns", "*", ":",
- "case-insensitive", "?", 0 };
- duplicate_rule( "Glob",
- bind_builtin( "GLOB", builtin_glob, 0, args ) );
- }
-
- {
- char const * args[] = { "patterns", "*", 0 };
- bind_builtin( "GLOB-RECURSIVELY",
- builtin_glob_recursive, 0, args );
- }
-
- duplicate_rule( "Includes",
- bind_builtin( "INCLUDES",
- builtin_depends, 1, 0 ) );
-
- {
- char const * args[] = { "targets", "*", ":", "targets-to-rebuild", "*",
- 0 };
- bind_builtin( "REBUILDS",
- builtin_rebuilds, 0, args );
- }
-
- duplicate_rule( "Leaves",
- bind_builtin( "LEAVES",
- builtin_flags, T_FLAG_LEAVES, 0 ) );
-
- duplicate_rule( "Match",
- bind_builtin( "MATCH",
- builtin_match, 0, 0 ) );
-
- {
- char const * args[] = { "string", ":", "delimiters", 0 };
- bind_builtin( "SPLIT_BY_CHARACTERS",
- builtin_split_by_characters, 0, args );
- }
-
- duplicate_rule( "NoCare",
- bind_builtin( "NOCARE",
- builtin_flags, T_FLAG_NOCARE, 0 ) );
-
- duplicate_rule( "NOTIME",
- duplicate_rule( "NotFile",
- bind_builtin( "NOTFILE",
- builtin_flags, T_FLAG_NOTFILE, 0 ) ) );
-
- duplicate_rule( "NoUpdate",
- bind_builtin( "NOUPDATE",
- builtin_flags, T_FLAG_NOUPDATE, 0 ) );
-
- duplicate_rule( "Temporary",
- bind_builtin( "TEMPORARY",
- builtin_flags, T_FLAG_TEMP, 0 ) );
-
- bind_builtin( "ISFILE",
- builtin_flags, T_FLAG_ISFILE, 0 );
-
- duplicate_rule( "HdrMacro",
- bind_builtin( "HDRMACRO",
- builtin_hdrmacro, 0, 0 ) );
-
- /* FAIL_EXPECTED is used to indicate that the result of a target build
- * action should be inverted (ok <=> fail) this can be useful when
- * performing test runs from Jamfiles.
- */
- bind_builtin( "FAIL_EXPECTED",
- builtin_flags, T_FLAG_FAIL_EXPECTED, 0 );
-
- bind_builtin( "RMOLD",
- builtin_flags, T_FLAG_RMOLD, 0 );
-
- {
- char const * args[] = { "targets", "*", 0 };
- bind_builtin( "UPDATE",
- builtin_update, 0, args );
- }
-
- {
- char const * args[] = { "targets", "*",
- ":", "log", "?",
- ":", "ignore-minus-n", "?",
- ":", "ignore-minus-q", "?", 0 };
- bind_builtin( "UPDATE_NOW",
- builtin_update_now, 0, args );
- }
-
- {
- char const * args[] = { "string", "pattern", "replacements", "+", 0 };
- duplicate_rule( "subst",
- bind_builtin( "SUBST",
- builtin_subst, 0, args ) );
- }
-
- {
- char const * args[] = { "module", "?", 0 };
- bind_builtin( "RULENAMES",
- builtin_rulenames, 0, args );
- }
-
- {
- char const * args[] = { "module", "?", 0 };
- bind_builtin( "VARNAMES",
- builtin_varnames, 0, args );
- }
-
- {
- char const * args[] = { "module", "?", 0 };
- bind_builtin( "DELETE_MODULE",
- builtin_delete_module, 0, args );
- }
-
- {
- char const * args[] = { "source_module", "?",
- ":", "source_rules", "*",
- ":", "target_module", "?",
- ":", "target_rules", "*",
- ":", "localize", "?", 0 };
- bind_builtin( "IMPORT",
- builtin_import, 0, args );
- }
-
- {
- char const * args[] = { "module", "?", ":", "rules", "*", 0 };
- bind_builtin( "EXPORT",
- builtin_export, 0, args );
- }
-
- {
- char const * args[] = { "levels", "?", 0 };
- bind_builtin( "CALLER_MODULE",
- builtin_caller_module, 0, args );
- }
-
- {
- char const * args[] = { "levels", "?", 0 };
- bind_builtin( "BACKTRACE",
- builtin_backtrace, 0, args );
- }
-
- {
- char const * args[] = { 0 };
- bind_builtin( "PWD",
- builtin_pwd, 0, args );
- }
-
- {
- char const * args[] = { "modules_to_import", "+",
- ":", "target_module", "?", 0 };
- bind_builtin( "IMPORT_MODULE",
- builtin_import_module, 0, args );
- }
-
- {
- char const * args[] = { "module", "?", 0 };
- bind_builtin( "IMPORTED_MODULES",
- builtin_imported_modules, 0, args );
- }
-
- {
- char const * args[] = { "instance_module", ":", "class_module", 0 };
- bind_builtin( "INSTANCE",
- builtin_instance, 0, args );
- }
-
- {
- char const * args[] = { "sequence", "*", 0 };
- bind_builtin( "SORT",
- builtin_sort, 0, args );
- }
-
- {
- char const * args[] = { "path_parts", "*", 0 };
- bind_builtin( "NORMALIZE_PATH",
- builtin_normalize_path, 0, args );
- }
-
- {
- char const * args[] = { "args", "*", 0 };
- bind_builtin( "CALC",
- builtin_calc, 0, args );
- }
-
- {
- char const * args[] = { "module", ":", "rule", 0 };
- bind_builtin( "NATIVE_RULE",
- builtin_native_rule, 0, args );
- }
-
- {
- char const * args[] = { "module", ":", "rule", ":", "version", 0 };
- bind_builtin( "HAS_NATIVE_RULE",
- builtin_has_native_rule, 0, args );
- }
-
- {
- char const * args[] = { "module", "*", 0 };
- bind_builtin( "USER_MODULE",
- builtin_user_module, 0, args );
- }
-
- {
- char const * args[] = { 0 };
- bind_builtin( "NEAREST_USER_LOCATION",
- builtin_nearest_user_location, 0, args );
- }
-
- {
- char const * args[] = { "file", 0 };
- bind_builtin( "CHECK_IF_FILE",
- builtin_check_if_file, 0, args );
- }
-
-#ifdef HAVE_PYTHON
- {
- char const * args[] = { "python-module",
- ":", "function",
- ":", "jam-module",
- ":", "rule-name", 0 };
- bind_builtin( "PYTHON_IMPORT_RULE",
- builtin_python_import_rule, 0, args );
- }
-#endif
-
-# if defined( OS_NT ) || defined( OS_CYGWIN )
- {
- char const * args[] = { "key_path", ":", "data", "?", 0 };
- bind_builtin( "W32_GETREG",
- builtin_system_registry, 0, args );
- }
-
- {
- char const * args[] = { "key_path", ":", "result-type", 0 };
- bind_builtin( "W32_GETREGNAMES",
- builtin_system_registry_names, 0, args );
- }
-# endif
-
- {
- char const * args[] = { "command", ":", "*", 0 };
- duplicate_rule( "SHELL",
- bind_builtin( "COMMAND",
- builtin_shell, 0, args ) );
- }
-
- {
- char const * args[] = { "string", 0 };
- bind_builtin( "MD5",
- builtin_md5, 0, args );
- }
-
- {
- char const * args[] = { "name", ":", "mode", 0 };
- bind_builtin( "FILE_OPEN",
- builtin_file_open, 0, args );
- }
-
- {
- char const * args[] = { "string", ":", "width", 0 };
- bind_builtin( "PAD",
- builtin_pad, 0, args );
- }
-
- {
- char const * args[] = { "targets", "*", 0 };
- bind_builtin( "PRECIOUS",
- builtin_precious, 0, args );
- }
-
- {
- char const * args [] = { 0 };
- bind_builtin( "SELF_PATH", builtin_self_path, 0, args );
- }
-
- {
- char const * args [] = { "path", 0 };
- bind_builtin( "MAKEDIR", builtin_makedir, 0, args );
- }
-
- /* Initialize builtin modules. */
- init_set();
- init_path();
- init_regex();
- init_property_set();
- init_sequence();
- init_order();
-}
-
-
-/*
- * builtin_calc() - CALC rule
- *
- * Performs simple mathematical operations on two arguments.
- */
-
-LIST * builtin_calc( FRAME * frame, int flags )
-{
- LIST * arg = lol_get( frame->args, 0 );
-
- LIST * result = L0;
- long lhs_value;
- long rhs_value;
- long result_value;
- char buffer[ 16 ];
- char const * lhs;
- char const * op;
- char const * rhs;
- LISTITER iter = list_begin( arg );
- LISTITER const end = list_end( arg );
-
- if ( iter == end ) return L0;
- lhs = object_str( list_item( iter ) );
-
- iter = list_next( iter );
- if ( iter == end ) return L0;
- op = object_str( list_item( iter ) );
-
- iter = list_next( iter );
- if ( iter == end ) return L0;
- rhs = object_str( list_item( iter ) );
-
- lhs_value = atoi( lhs );
- rhs_value = atoi( rhs );
-
- if ( !strcmp( "+", op ) )
- result_value = lhs_value + rhs_value;
- else if ( !strcmp( "-", op ) )
- result_value = lhs_value - rhs_value;
- else
- return L0;
-
- sprintf( buffer, "%ld", result_value );
- result = list_push_back( result, object_new( buffer ) );
- return result;
-}
-
-
-/*
- * builtin_depends() - DEPENDS/INCLUDES rule
- *
- * The DEPENDS/INCLUDES builtin rule appends each of the listed sources on the
- * dependency/includes list of each of the listed targets. It binds both the
- * targets and sources as TARGETs.
- */
-
-LIST * builtin_depends( FRAME * frame, int flags )
-{
- LIST * const targets = lol_get( frame->args, 0 );
- LIST * const sources = lol_get( frame->args, 1 );
-
- LISTITER iter = list_begin( targets );
- LISTITER end = list_end( targets );
- for ( ; iter != end; iter = list_next( iter ) )
- {
- TARGET * const t = bindtarget( list_item( iter ) );
-
- if ( flags )
- target_include_many( t, sources );
- else
- t->depends = targetlist( t->depends, sources );
- }
-
- /* Enter reverse links */
- iter = list_begin( sources );
- end = list_end( sources );
- for ( ; iter != end; iter = list_next( iter ) )
- {
- TARGET * const s = bindtarget( list_item( iter ) );
- if ( flags )
- {
- LISTITER t_iter = list_begin( targets );
- LISTITER const t_end = list_end( targets );
- for ( ; t_iter != t_end; t_iter = list_next( t_iter ) )
- s->dependants = targetentry( s->dependants, bindtarget(
- list_item( t_iter ) )->includes );
- }
- else
- s->dependants = targetlist( s->dependants, targets );
- }
-
- return L0;
-}
-
-
-/*
- * builtin_rebuilds() - REBUILDS rule
- *
- * Appends each of the rebuild-targets listed in its second argument to the
- * rebuilds list for each of the targets listed in its first argument.
- */
-
-LIST * builtin_rebuilds( FRAME * frame, int flags )
-{
- LIST * targets = lol_get( frame->args, 0 );
- LIST * rebuilds = lol_get( frame->args, 1 );
- LISTITER iter = list_begin( targets );
- LISTITER const end = list_end( targets );
- for ( ; iter != end; iter = list_next( iter ) )
- {
- TARGET * const t = bindtarget( list_item( iter ) );
- t->rebuilds = targetlist( t->rebuilds, rebuilds );
- }
- return L0;
-}
-
-
-/*
- * builtin_echo() - ECHO rule
- *
- * Echoes the targets to the user. No other actions are taken.
- */
-
-LIST * builtin_echo( FRAME * frame, int flags )
-{
- list_print( lol_get( frame->args, 0 ) );
- printf( "\n" );
- fflush( stdout );
- return L0;
-}
-
-
-/*
- * builtin_exit() - EXIT rule
- *
- * Echoes the targets to the user and exits the program with a failure status.
- */
-
-LIST * builtin_exit( FRAME * frame, int flags )
-{
- LIST * const code = lol_get( frame->args, 1 );
- list_print( lol_get( frame->args, 0 ) );
- printf( "\n" );
- if ( !list_empty( code ) )
- exit( atoi( object_str( list_front( code ) ) ) );
- else
- exit( EXITBAD ); /* yeech */
- return L0;
-}
-
-
-/*
- * builtin_flags() - NOCARE, NOTFILE, TEMPORARY rule
- *
- * Marks the target with the appropriate flag, for use by make0(). It binds each
- * target as a TARGET.
- */
-
-LIST * builtin_flags( FRAME * frame, int flags )
-{
- LIST * const targets = lol_get( frame->args, 0 );
- LISTITER iter = list_begin( targets );
- LISTITER const end = list_end( targets );
- for ( ; iter != end; iter = list_next( iter ) )
- bindtarget( list_item( iter ) )->flags |= flags;
- return L0;
-}
-
-
-/*
- * builtin_glob() - GLOB rule
- */
-
-struct globbing
-{
- LIST * patterns;
- LIST * results;
- LIST * case_insensitive;
-};
-
-
-static void downcase_inplace( char * p )
-{
- for ( ; *p; ++p )
- *p = tolower( *p );
-}
-
-
-static void builtin_glob_back( void * closure, OBJECT * file, int status,
- timestamp const * const time )
-{
- PROFILE_ENTER( BUILTIN_GLOB_BACK );
-
- struct globbing * const globbing = (struct globbing *)closure;
- PATHNAME f;
- string buf[ 1 ];
- LISTITER iter;
- LISTITER end;
-
- /* Null out directory for matching. We wish we had file_dirscan() pass up a
- * PATHNAME.
- */
- path_parse( object_str( file ), &f );
- f.f_dir.len = 0;
-
- /* For globbing, we unconditionally ignore current and parent directory
- * items. Since these items always exist, there is no reason why caller of
- * GLOB would want to see them. We could also change file_dirscan(), but
- * then paths with embedded "." and ".." would not work anywhere.
- */
- if ( !strcmp( f.f_base.ptr, "." ) || !strcmp( f.f_base.ptr, ".." ) )
- {
- PROFILE_EXIT( BUILTIN_GLOB_BACK );
- return;
- }
-
- string_new( buf );
- path_build( &f, buf );
-
- if ( globbing->case_insensitive )
- downcase_inplace( buf->value );
-
- iter = list_begin( globbing->patterns );
- end = list_end( globbing->patterns );
- for ( ; iter != end; iter = list_next( iter ) )
- {
- if ( !glob( object_str( list_item( iter ) ), buf->value ) )
- {
- globbing->results = list_push_back( globbing->results, object_copy(
- file ) );
- break;
- }
- }
-
- string_free( buf );
-
- PROFILE_EXIT( BUILTIN_GLOB_BACK );
-}
-
-
-static LIST * downcase_list( LIST * in )
-{
- LIST * result = L0;
- LISTITER iter = list_begin( in );
- LISTITER const end = list_end( in );
-
- string s[ 1 ];
- string_new( s );
-
- for ( ; iter != end; iter = list_next( iter ) )
- {
- string_append( s, object_str( list_item( iter ) ) );
- downcase_inplace( s->value );
- result = list_push_back( result, object_new( s->value ) );
- string_truncate( s, 0 );
- }
-
- string_free( s );
- return result;
-}
-
-
-LIST * builtin_glob( FRAME * frame, int flags )
-{
- LIST * const l = lol_get( frame->args, 0 );
- LIST * const r = lol_get( frame->args, 1 );
-
- LISTITER iter;
- LISTITER end;
- struct globbing globbing;
-
- globbing.results = L0;
- globbing.patterns = r;
-
- globbing.case_insensitive =
-# if defined( OS_NT ) || defined( OS_CYGWIN )
- l; /* Always case-insensitive if any files can be found. */
-# else
- lol_get( frame->args, 2 );
-# endif
-
- if ( globbing.case_insensitive )
- globbing.patterns = downcase_list( r );
-
- iter = list_begin( l );
- end = list_end( l );
- for ( ; iter != end; iter = list_next( iter ) )
- file_dirscan( list_item( iter ), builtin_glob_back, &globbing );
-
- if ( globbing.case_insensitive )
- list_free( globbing.patterns );
-
- return globbing.results;
-}
-
-
-static int has_wildcards( char const * const str )
-{
- return str[ strcspn( str, "[]*?" ) ] ? 1 : 0;
-}
-
-
-/*
- * append_if_exists() - if file exists, append it to the list
- */
-
-static LIST * append_if_exists( LIST * list, OBJECT * file )
-{
- return file_query( file )
- ? list_push_back( list, object_copy( file ) )
- : list ;
-}
-
-
-LIST * glob1( OBJECT * dirname, OBJECT * pattern )
-{
- LIST * const plist = list_new( object_copy( pattern ) );
- struct globbing globbing;
-
- globbing.results = L0;
- globbing.patterns = plist;
-
- globbing.case_insensitive
-# if defined( OS_NT ) || defined( OS_CYGWIN )
- = plist; /* always case-insensitive if any files can be found */
-# else
- = L0;
-# endif
-
- if ( globbing.case_insensitive )
- globbing.patterns = downcase_list( plist );
-
- file_dirscan( dirname, builtin_glob_back, &globbing );
-
- if ( globbing.case_insensitive )
- list_free( globbing.patterns );
-
- list_free( plist );
-
- return globbing.results;
-}
-
-
-LIST * glob_recursive( char const * pattern )
-{
- LIST * result = L0;
-
- /* Check if there's metacharacters in pattern */
- if ( !has_wildcards( pattern ) )
- {
- /* No metacharacters. Check if the path exists. */
- OBJECT * const p = object_new( pattern );
- result = append_if_exists( result, p );
- object_free( p );
- }
- else
- {
- /* Have metacharacters in the pattern. Split into dir/name. */
- PATHNAME path[ 1 ];
- path_parse( pattern, path );
-
- if ( path->f_dir.ptr )
- {
- LIST * dirs = L0;
- string dirname[ 1 ];
- string basename[ 1 ];
- string_new( dirname );
- string_new( basename );
-
- string_append_range( dirname, path->f_dir.ptr,
- path->f_dir.ptr + path->f_dir.len );
-
- path->f_grist.ptr = 0;
- path->f_grist.len = 0;
- path->f_dir.ptr = 0;
- path->f_dir.len = 0;
- path_build( path, basename );
-
- dirs = has_wildcards( dirname->value )
- ? glob_recursive( dirname->value )
- : list_push_back( dirs, object_new( dirname->value ) );
-
- if ( has_wildcards( basename->value ) )
- {
- OBJECT * const b = object_new( basename->value );
- LISTITER iter = list_begin( dirs );
- LISTITER const end = list_end( dirs );
- for ( ; iter != end; iter = list_next( iter ) )
- result = list_append( result, glob1( list_item( iter ), b )
- );
- object_free( b );
- }
- else
- {
- LISTITER iter = list_begin( dirs );
- LISTITER const end = list_end( dirs );
- string file_string[ 1 ];
- string_new( file_string );
-
- /* No wildcard in basename. */
- for ( ; iter != end; iter = list_next( iter ) )
- {
- OBJECT * p;
- path->f_dir.ptr = object_str( list_item( iter ) );
- path->f_dir.len = strlen( object_str( list_item( iter ) ) );
- path_build( path, file_string );
-
- p = object_new( file_string->value );
-
- result = append_if_exists( result, p );
-
- object_free( p );
-
- string_truncate( file_string, 0 );
- }
-
- string_free( file_string );
- }
-
- string_free( dirname );
- string_free( basename );
-
- list_free( dirs );
- }
- else
- {
- /* No directory, just a pattern. */
- OBJECT * const p = object_new( pattern );
- result = list_append( result, glob1( constant_dot, p ) );
- object_free( p );
- }
- }
-
- return result;
-}
-
-
-/*
- * builtin_glob_recursive() - ???
- */
-
-LIST * builtin_glob_recursive( FRAME * frame, int flags )
-{
- LIST * result = L0;
- LIST * const l = lol_get( frame->args, 0 );
- LISTITER iter = list_begin( l );
- LISTITER const end = list_end( l );
- for ( ; iter != end; iter = list_next( iter ) )
- result = list_append( result, glob_recursive( object_str( list_item(
- iter ) ) ) );
- return result;
-}
-
-
-/*
- * builtin_match() - MATCH rule, regexp matching
- */
-
-LIST * builtin_match( FRAME * frame, int flags )
-{
- LIST * l;
- LIST * r;
- LIST * result = L0;
- LISTITER l_iter;
- LISTITER l_end;
- LISTITER r_iter;
- LISTITER r_end;
-
- string buf[ 1 ];
- string_new( buf );
-
- /* For each pattern */
-
- l = lol_get( frame->args, 0 );
- l_iter = list_begin( l );
- l_end = list_end( l );
- for ( ; l_iter != l_end; l_iter = list_next( l_iter ) )
- {
- /* Result is cached and intentionally never freed. */
- regexp * re = regex_compile( list_item( l_iter ) );
-
- /* For each string to match against. */
- r = lol_get( frame->args, 1 );
- r_iter = list_begin( r );
- r_end = list_end( r );
- for ( ; r_iter != r_end; r_iter = list_next( r_iter ) )
- {
- if ( regexec( re, object_str( list_item( r_iter ) ) ) )
- {
- int i;
- int top;
-
- /* Find highest parameter */
-
- for ( top = NSUBEXP; top-- > 1; )
- if ( re->startp[ top ] )
- break;
-
- /* And add all parameters up to highest onto list. */
- /* Must have parameters to have results! */
- for ( i = 1; i <= top; ++i )
- {
- string_append_range( buf, re->startp[ i ], re->endp[ i ] );
- result = list_push_back( result, object_new( buf->value ) );
- string_truncate( buf, 0 );
- }
- }
- }
- }
-
- string_free( buf );
- return result;
-}
-
-
-/*
- * builtin_split_by_characters() - splits the given string into tokens
- */
-
-LIST * builtin_split_by_characters( FRAME * frame, int flags )
-{
- LIST * l1 = lol_get( frame->args, 0 );
- LIST * l2 = lol_get( frame->args, 1 );
-
- LIST * result = L0;
-
- string buf[ 1 ];
-
- char const * delimiters = object_str( list_front( l2 ) );
- char * t;
-
- string_copy( buf, object_str( list_front( l1 ) ) );
-
- t = strtok( buf->value, delimiters );
- while ( t )
- {
- result = list_push_back( result, object_new( t ) );
- t = strtok( NULL, delimiters );
- }
-
- string_free( buf );
-
- return result;
-}
-
-
-/*
- * builtin_hdrmacro() - ???
- */
-
-LIST * builtin_hdrmacro( FRAME * frame, int flags )
-{
- LIST * const l = lol_get( frame->args, 0 );
- LISTITER iter = list_begin( l );
- LISTITER const end = list_end( l );
-
- for ( ; iter != end; iter = list_next( iter ) )
- {
- TARGET * const t = bindtarget( list_item( iter ) );
-
- /* Scan file for header filename macro definitions. */
- if ( DEBUG_HEADER )
- printf( "scanning '%s' for header file macro definitions\n",
- object_str( list_item( iter ) ) );
-
- macro_headers( t );
- }
-
- return L0;
-}
-
-
-/*
- * builtin_rulenames() - RULENAMES ( MODULE ? )
- *
- * Returns a list of the non-local rule names in the given MODULE. If MODULE is
- * not supplied, returns the list of rule names in the global module.
- */
-
-static void add_rule_name( void * r_, void * result_ )
-{
- RULE * const r = (RULE *)r_;
- LIST * * const result = (LIST * *)result_;
- if ( r->exported )
- *result = list_push_back( *result, object_copy( r->name ) );
-}
-
-
-LIST * builtin_rulenames( FRAME * frame, int flags )
-{
- LIST * arg0 = lol_get( frame->args, 0 );
- LIST * result = L0;
- module_t * const source_module = bindmodule( list_empty( arg0 )
- ? 0
- : list_front( arg0 ) );
-
- if ( source_module->rules )
- hashenumerate( source_module->rules, add_rule_name, &result );
- return result;
-}
-
-
-/*
- * builtin_varnames() - VARNAMES ( MODULE ? )
- *
- * Returns a list of the variable names in the given MODULE. If MODULE is not
- * supplied, returns the list of variable names in the global module.
- */
-
-/* helper function for builtin_varnames(), below. Used with hashenumerate, will
- * prepend the key of each element to the list
- */
-static void add_hash_key( void * np, void * result_ )
-{
- LIST * * result = (LIST * *)result_;
- *result = list_push_back( *result, object_copy( *(OBJECT * *)np ) );
-}
-
-
-LIST * builtin_varnames( FRAME * frame, int flags )
-{
- LIST * arg0 = lol_get( frame->args, 0 );
- LIST * result = L0;
- module_t * source_module = bindmodule( list_empty( arg0 )
- ? 0
- : list_front( arg0 ) );
-
- struct hash * const vars = source_module->variables;
- if ( vars )
- hashenumerate( vars, add_hash_key, &result );
- return result;
-}
-
-
-/*
- * builtin_delete_module() - DELETE_MODULE ( MODULE ? )
- *
- * Clears all rules and variables from the given module.
- */
-
-LIST * builtin_delete_module( FRAME * frame, int flags )
-{
- LIST * const arg0 = lol_get( frame->args, 0 );
- module_t * const source_module = bindmodule( list_empty( arg0 ) ? 0 :
- list_front( arg0 ) );
- delete_module( source_module );
- return L0;
-}
-
-
-/*
- * unknown_rule() - reports an unknown rule occurrence to the user and exits
- */
-
-void unknown_rule( FRAME * frame, char const * key, module_t * module,
- OBJECT * rule_name )
-{
- backtrace_line( frame->prev );
- if ( key )
- printf("%s error", key);
- else
- printf("ERROR");
- printf( ": rule \"%s\" unknown in ", object_str( rule_name ) );
- if ( module->name )
- printf( "module \"%s\".\n", object_str( module->name ) );
- else
- printf( "root module.\n" );
- backtrace( frame->prev );
- exit( 1 );
-}
-
-
-/*
- * builtin_import() - IMPORT rule
- *
- * IMPORT
- * (
- * SOURCE_MODULE ? :
- * SOURCE_RULES * :
- * TARGET_MODULE ? :
- * TARGET_RULES * :
- * LOCALIZE ?
- * )
- *
- * Imports rules from the SOURCE_MODULE into the TARGET_MODULE as local rules.
- * If either SOURCE_MODULE or TARGET_MODULE is not supplied, it refers to the
- * global module. SOURCE_RULES specifies which rules from the SOURCE_MODULE to
- * import; TARGET_RULES specifies the names to give those rules in
- * TARGET_MODULE. If SOURCE_RULES contains a name that does not correspond to
- * a rule in SOURCE_MODULE, or if it contains a different number of items than
- * TARGET_RULES, an error is issued. If LOCALIZE is specified, the rules will be
- * executed in TARGET_MODULE, with corresponding access to its module local
- * variables.
- */
-
-LIST * builtin_import( FRAME * frame, int flags )
-{
- LIST * source_module_list = lol_get( frame->args, 0 );
- LIST * source_rules = lol_get( frame->args, 1 );
- LIST * target_module_list = lol_get( frame->args, 2 );
- LIST * target_rules = lol_get( frame->args, 3 );
- LIST * localize = lol_get( frame->args, 4 );
-
- module_t * target_module = bindmodule( list_empty( target_module_list )
- ? 0
- : list_front( target_module_list ) );
- module_t * source_module = bindmodule( list_empty( source_module_list )
- ? 0
- : list_front( source_module_list ) );
-
- LISTITER source_iter = list_begin( source_rules );
- LISTITER const source_end = list_end( source_rules );
- LISTITER target_iter = list_begin( target_rules );
- LISTITER const target_end = list_end( target_rules );
-
- for ( ;
- source_iter != source_end && target_iter != target_end;
- source_iter = list_next( source_iter ),
- target_iter = list_next( target_iter ) )
- {
- RULE * r;
- RULE * imported;
-
- if ( !source_module->rules || !(r = (RULE *)hash_find(
- source_module->rules, list_item( source_iter ) ) ) )
- unknown_rule( frame, "IMPORT", source_module, list_item( source_iter
- ) );
-
- imported = import_rule( r, target_module, list_item( target_iter ) );
- if ( !list_empty( localize ) )
- rule_localize( imported, target_module );
- /* This rule is really part of some other module. Just refer to it here,
- * but do not let it out.
- */
- imported->exported = 0;
- }
-
- if ( source_iter != source_end || target_iter != target_end )
- {
- backtrace_line( frame->prev );
- printf( "import error: length of source and target rule name lists "
- "don't match!\n" );
- printf( " source: " );
- list_print( source_rules );
- printf( "\n target: " );
- list_print( target_rules );
- printf( "\n" );
- backtrace( frame->prev );
- exit( 1 );
- }
-
- return L0;
-}
-
-
-/*
- * builtin_export() - EXPORT ( MODULE ? : RULES * )
- *
- * The EXPORT rule marks RULES from the SOURCE_MODULE as non-local (and thus
- * exportable). If an element of RULES does not name a rule in MODULE, an error
- * is issued.
- */
-
-LIST * builtin_export( FRAME * frame, int flags )
-{
- LIST * const module_list = lol_get( frame->args, 0 );
- LIST * const rules = lol_get( frame->args, 1 );
- module_t * const m = bindmodule( list_empty( module_list ) ? 0 : list_front(
- module_list ) );
-
- LISTITER iter = list_begin( rules );
- LISTITER const end = list_end( rules );
- for ( ; iter != end; iter = list_next( iter ) )
- {
- RULE * r;
- if ( !m->rules || !( r = (RULE *)hash_find( m->rules, list_item( iter )
- ) ) )
- unknown_rule( frame, "EXPORT", m, list_item( iter ) );
- r->exported = 1;
- }
- return L0;
-}
-
-
-/*
- * get_source_line() - get a frame's file and line number information
- *
- * This is the execution traceback information to be indicated for in debug
- * output or an error backtrace.
- */
-
-static void get_source_line( FRAME * frame, char const * * file, int * line )
-{
- if ( frame->file )
- {
- char const * f = object_str( frame->file );
- int l = frame->line;
- if ( !strcmp( f, "+" ) )
- {
- f = "jambase.c";
- l += 3;
- }
- *file = f;
- *line = l;
- }
- else
- {
- *file = "(builtin)";
- *line = -1;
- }
-}
-
-
-void print_source_line( FRAME * frame )
-{
- char const * file;
- int line;
- get_source_line( frame, &file, &line );
- if ( line < 0 )
- printf( "(builtin):" );
- else
- printf( "%s:%d:", file, line );
-}
-
-
-/*
- * backtrace_line() - print a single line of error backtrace for the given
- * frame.
- */
-
-void backtrace_line( FRAME * frame )
-{
- if ( frame == 0 )
- {
- printf( "(no frame):" );
- }
- else
- {
- print_source_line( frame );
- printf( " in %s\n", frame->rulename );
- }
-}
-
-
-/*
- * backtrace() - Print the entire backtrace from the given frame to the Jambase
- * which invoked it.
- */
-
-void backtrace( FRAME * frame )
-{
- if ( !frame ) return;
- while ( ( frame = frame->prev ) )
- backtrace_line( frame );
-}
-
-
-/*
- * builtin_backtrace() - A Jam version of the backtrace function, taking no
- * arguments and returning a list of quadruples: FILENAME LINE MODULE. RULENAME
- * describing each frame. Note that the module-name is always followed by a
- * period.
- */
-
-LIST * builtin_backtrace( FRAME * frame, int flags )
-{
- LIST * const levels_arg = lol_get( frame->args, 0 );
- int levels = list_empty( levels_arg )
- ? (int)( (unsigned int)(-1) >> 1 )
- : atoi( object_str( list_front( levels_arg ) ) );
-
- LIST * result = L0;
- for ( ; ( frame = frame->prev ) && levels; --levels )
- {
- char const * file;
- int line;
- char buf[ 32 ];
- string module_name[ 1 ];
- get_source_line( frame, &file, &line );
- sprintf( buf, "%d", line );
- string_new( module_name );
- if ( frame->module->name )
- {
- string_append( module_name, object_str( frame->module->name ) );
- string_append( module_name, "." );
- }
- result = list_push_back( result, object_new( file ) );
- result = list_push_back( result, object_new( buf ) );
- result = list_push_back( result, object_new( module_name->value ) );
- result = list_push_back( result, object_new( frame->rulename ) );
- string_free( module_name );
- }
- return result;
-}
-
-
-/*
- * builtin_caller_module() - CALLER_MODULE ( levels ? )
- *
- * If levels is not supplied, returns the name of the module of the rule which
- * called the one calling this one. If levels is supplied, it is interpreted as
- * an integer specifying a number of additional levels of call stack to traverse
- * in order to locate the module in question. If no such module exists, returns
- * the empty list. Also returns the empty list when the module in question is
- * the global module. This rule is needed for implementing module import
- * behavior.
- */
-
-LIST * builtin_caller_module( FRAME * frame, int flags )
-{
- LIST * const levels_arg = lol_get( frame->args, 0 );
- int const levels = list_empty( levels_arg )
- ? 0
- : atoi( object_str( list_front( levels_arg ) ) );
-
- int i;
- for ( i = 0; ( i < levels + 2 ) && frame->prev; ++i )
- frame = frame->prev;
-
- return frame->module == root_module()
- ? L0
- : list_new( object_copy( frame->module->name ) );
-}
-
-
-/*
- * Return the current working directory.
- *
- * Usage: pwd = [ PWD ] ;
- */
-
-LIST * builtin_pwd( FRAME * frame, int flags )
-{
- return list_new( object_copy( cwd() ) );
-}
-
-
-/*
- * Adds targets to the list of target that jam will attempt to update.
- */
-
-LIST * builtin_update( FRAME * frame, int flags )
-{
- LIST * result = list_copy( targets_to_update() );
- LIST * arg1 = lol_get( frame->args, 0 );
- LISTITER iter = list_begin( arg1 ), end = list_end( arg1 );
- clear_targets_to_update();
- for ( ; iter != end; iter = list_next( iter ) )
- mark_target_for_updating( object_copy( list_item( iter ) ) );
- return result;
-}
-
-extern int anyhow;
-int last_update_now_status;
-
-/* Takes a list of target names and immediately updates them.
- *
- * Parameters:
- * 1. Target list.
- * 2. Optional file descriptor (converted to a string) for a log file where all
- * the related build output should be redirected.
- * 3. If specified, makes the build temporarily disable the -n option, i.e.
- * forces all needed out-of-date targets to be rebuilt.
- * 4. If specified, makes the build temporarily disable the -q option, i.e.
- * forces the build to continue even if one of the targets fails to build.
- */
-LIST * builtin_update_now( FRAME * frame, int flags )
-{
- LIST * targets = lol_get( frame->args, 0 );
- LIST * log = lol_get( frame->args, 1 );
- LIST * force = lol_get( frame->args, 2 );
- LIST * continue_ = lol_get( frame->args, 3 );
- int status;
- int original_stdout = 0;
- int original_stderr = 0;
- int original_noexec = 0;
- int original_quitquick = 0;
-
- if ( !list_empty( log ) )
- {
- /* Temporarily redirect stdout and stderr to the given log file. */
- int const fd = atoi( object_str( list_front( log ) ) );
- original_stdout = dup( 0 );
- original_stderr = dup( 1 );
- dup2( fd, 0 );
- dup2( fd, 1 );
- }
-
- if ( !list_empty( force ) )
- {
- original_noexec = globs.noexec;
- globs.noexec = 0;
- }
-
- if ( !list_empty( continue_ ) )
- {
- original_quitquick = globs.quitquick;
- globs.quitquick = 0;
- }
-
- status = make( targets, anyhow );
-
- if ( !list_empty( force ) )
- {
- globs.noexec = original_noexec;
- }
-
- if ( !list_empty( continue_ ) )
- {
- globs.quitquick = original_quitquick;
- }
-
- if ( !list_empty( log ) )
- {
- /* Flush whatever stdio might have buffered, while descriptions 0 and 1
- * still refer to the log file.
- */
- fflush( stdout );
- fflush( stderr );
- dup2( original_stdout, 0 );
- dup2( original_stderr, 1 );
- close( original_stdout );
- close( original_stderr );
- }
-
- last_update_now_status = status;
-
- return status ? L0 : list_new( object_copy( constant_ok ) );
-}
-
-
-LIST * builtin_import_module( FRAME * frame, int flags )
-{
- LIST * const arg1 = lol_get( frame->args, 0 );
- LIST * const arg2 = lol_get( frame->args, 1 );
- module_t * const m = list_empty( arg2 )
- ? root_module()
- : bindmodule( list_front( arg2 ) );
- import_module( arg1, m );
- return L0;
-}
-
-
-LIST * builtin_imported_modules( FRAME * frame, int flags )
-{
- LIST * const arg0 = lol_get( frame->args, 0 );
- OBJECT * const module = list_empty( arg0 ) ? 0 : list_front( arg0 );
- return imported_modules( bindmodule( module ) );
-}
-
-
-LIST * builtin_instance( FRAME * frame, int flags )
-{
- LIST * arg1 = lol_get( frame->args, 0 );
- LIST * arg2 = lol_get( frame->args, 1 );
- module_t * const instance = bindmodule( list_front( arg1 ) );
- module_t * const class_module = bindmodule( list_front( arg2 ) );
- instance->class_module = class_module;
- module_set_fixed_variables( instance, class_module->num_fixed_variables );
- return L0;
-}
-
-
-LIST * builtin_sort( FRAME * frame, int flags )
-{
- return list_sort( lol_get( frame->args, 0 ) );
-}
-
-
-LIST * builtin_normalize_path( FRAME * frame, int flags )
-{
- LIST * arg = lol_get( frame->args, 0 );
-
- /* First, we iterate over all '/'-separated elements, starting from the end
- * of string. If we see a '..', we remove a preceeding path element. If we
- * see '.', we remove it. Removal is done by overwriting data using '\1'
- * characters. After the whole string has been processed, we do a second
- * pass, removing any entered '\1' characters.
- */
-
- string in[ 1 ];
- string out[ 1 ];
- /* Last character of the part of string still to be processed. */
- char * end;
- /* Working pointer. */
- char * current;
- /* Number of '..' elements seen and not processed yet. */
- int dotdots = 0;
- int rooted = 0;
- OBJECT * result = 0;
- LISTITER arg_iter = list_begin( arg );
- LISTITER arg_end = list_end( arg );
-
- /* Make a copy of input: we should not change it. Prepend a '/' before it as
- * a guard for the algorithm later on and remember whether it was originally
- * rooted or not.
- */
- string_new( in );
- string_push_back( in, '/' );
- for ( ; arg_iter != arg_end; arg_iter = list_next( arg_iter ) )
- {
- if ( object_str( list_item( arg_iter ) )[ 0 ] != '\0' )
- {
- if ( in->size == 1 )
- rooted = ( object_str( list_item( arg_iter ) )[ 0 ] == '/' ) ||
- ( object_str( list_item( arg_iter ) )[ 0 ] == '\\' );
- else
- string_append( in, "/" );
- string_append( in, object_str( list_item( arg_iter ) ) );
- }
- }
-
- /* Convert \ into /. On Windows, paths using / and \ are equivalent, and we
- * want this function to obtain a canonic representation.
- */
- for ( current = in->value, end = in->value + in->size;
- current < end; ++current )
- if ( *current == '\\' )
- *current = '/';
-
- /* Now we remove any extra path elements by overwriting them with '\1'
- * characters and cound how many more unused '..' path elements there are
- * remaining. Note that each remaining path element with always starts with
- * a '/' character.
- */
- for ( end = in->value + in->size - 1; end >= in->value; )
- {
- /* Set 'current' to the next occurence of '/', which always exists. */
- for ( current = end; *current != '/'; --current );
-
- if ( current == end )
- {
- /* Found a trailing or duplicate '/'. Remove it. */
- *current = '\1';
- }
- else if ( ( end - current == 1 ) && ( *( current + 1 ) == '.' ) )
- {
- /* Found '/.'. Remove them all. */
- *current = '\1';
- *(current + 1) = '\1';
- }
- else if ( ( end - current == 2 ) && ( *( current + 1 ) == '.' ) &&
- ( *( current + 2 ) == '.' ) )
- {
- /* Found '/..'. Remove them all. */
- *current = '\1';
- *(current + 1) = '\1';
- *(current + 2) = '\1';
- ++dotdots;
- }
- else if ( dotdots )
- {
- memset( current, '\1', end - current + 1 );
- --dotdots;
- }
- end = current - 1;
- }
-
- string_new( out );
-
- /* Now we know that we need to add exactly dotdots '..' path elements to the
- * front and that our string is either empty or has a '/' as its first
- * significant character. If we have any dotdots remaining then the passed
- * path must not have been rooted or else it is invalid we return an empty
- * list.
- */
- if ( dotdots )
- {
- if ( rooted )
- {
- string_free( out );
- string_free( in );
- return L0;
- }
- do
- string_append( out, "/.." );
- while ( --dotdots );
- }
-
- /* Now we actually remove all the path characters marked for removal. */
- for ( current = in->value; *current; ++current )
- if ( *current != '\1' )
- string_push_back( out, *current );
-
- /* Here we know that our string contains no '\1' characters and is either
- * empty or has a '/' as its initial character. If the original path was not
- * rooted and we have a non-empty path we need to drop the initial '/'. If
- * the original path was rooted and we have an empty path we need to add
- * back the '/'.
- */
- result = object_new( out->size
- ? out->value + !rooted
- : ( rooted ? "/" : "." ) );
-
- string_free( out );
- string_free( in );
-
- return list_new( result );
-}
-
-
-LIST * builtin_native_rule( FRAME * frame, int flags )
-{
- LIST * module_name = lol_get( frame->args, 0 );
- LIST * rule_name = lol_get( frame->args, 1 );
-
- module_t * module = bindmodule( list_front( module_name ) );
-
- native_rule_t * np;
- if ( module->native_rules && (np = (native_rule_t *)hash_find(
- module->native_rules, list_front( rule_name ) ) ) )
- {
- new_rule_body( module, np->name, np->procedure, 1 );
- }
- else
- {
- backtrace_line( frame->prev );
- printf( "error: no native rule \"%s\" defined in module \"%s.\"\n",
- object_str( list_front( rule_name ) ), object_str( module->name ) );
- backtrace( frame->prev );
- exit( 1 );
- }
- return L0;
-}
-
-
-LIST * builtin_has_native_rule( FRAME * frame, int flags )
-{
- LIST * module_name = lol_get( frame->args, 0 );
- LIST * rule_name = lol_get( frame->args, 1 );
- LIST * version = lol_get( frame->args, 2 );
-
- module_t * module = bindmodule( list_front( module_name ) );
-
- native_rule_t * np;
- if ( module->native_rules && (np = (native_rule_t *)hash_find(
- module->native_rules, list_front( rule_name ) ) ) )
- {
- int expected_version = atoi( object_str( list_front( version ) ) );
- if ( np->version == expected_version )
- return list_new( object_copy( constant_true ) );
- }
- return L0;
-}
-
-
-LIST * builtin_user_module( FRAME * frame, int flags )
-{
- LIST * const module_name = lol_get( frame->args, 0 );
- LISTITER iter = list_begin( module_name );
- LISTITER const end = list_end( module_name );
- for ( ; iter != end; iter = list_next( iter ) )
- bindmodule( list_item( iter ) )->user_module = 1;
- return L0;
-}
-
-
-LIST * builtin_nearest_user_location( FRAME * frame, int flags )
-{
- FRAME * const nearest_user_frame = frame->module->user_module
- ? frame
- : frame->prev_user;
- if ( !nearest_user_frame )
- return L0;
-
- {
- LIST * result = L0;
- char const * file;
- int line;
- char buf[ 32 ];
-
- get_source_line( nearest_user_frame, &file, &line );
- sprintf( buf, "%d", line );
- result = list_push_back( result, object_new( file ) );
- result = list_push_back( result, object_new( buf ) );
- return result;
- }
-}
-
-
-LIST * builtin_check_if_file( FRAME * frame, int flags )
-{
- LIST * const name = lol_get( frame->args, 0 );
- return file_is_file( list_front( name ) ) == 1
- ? list_new( object_copy( constant_true ) )
- : L0;
-}
-
-
-LIST * builtin_md5( FRAME * frame, int flags )
-{
- LIST * l = lol_get( frame->args, 0 );
- char const * s = object_str( list_front( l ) );
-
- md5_state_t state;
- md5_byte_t digest[ 16 ];
- char hex_output[ 16 * 2 + 1 ];
-
- int di;
-
- md5_init( &state );
- md5_append( &state, (md5_byte_t const *)s, strlen( s ) );
- md5_finish( &state, digest );
-
- for ( di = 0; di < 16; ++di )
- sprintf( hex_output + di * 2, "%02x", digest[ di ] );
-
- return list_new( object_new( hex_output ) );
-}
-
-
-LIST * builtin_file_open( FRAME * frame, int flags )
-{
- char const * name = object_str( list_front( lol_get( frame->args, 0 ) ) );
- char const * mode = object_str( list_front( lol_get( frame->args, 1 ) ) );
- int fd;
- char buffer[ sizeof( "4294967295" ) ];
-
- if ( strcmp(mode, "w") == 0 )
- fd = open( name, O_WRONLY|O_CREAT|O_TRUNC, 0666 );
- else
- fd = open( name, O_RDONLY );
-
- if ( fd != -1 )
- {
- sprintf( buffer, "%d", fd );
- return list_new( object_new( buffer ) );
- }
- return L0;
-}
-
-
-LIST * builtin_pad( FRAME * frame, int flags )
-{
- OBJECT * string = list_front( lol_get( frame->args, 0 ) );
- char const * width_s = object_str( list_front( lol_get( frame->args, 1 ) ) );
-
- int current = strlen( object_str( string ) );
- int desired = atoi( width_s );
- if ( current >= desired )
- return list_new( object_copy( string ) );
- else
- {
- char * buffer = BJAM_MALLOC( desired + 1 );
- int i;
- LIST * result;
-
- strcpy( buffer, object_str( string ) );
- for ( i = current; i < desired; ++i )
- buffer[ i ] = ' ';
- buffer[ desired ] = '\0';
- result = list_new( object_new( buffer ) );
- BJAM_FREE( buffer );
- return result;
- }
-}
-
-
-LIST * builtin_precious( FRAME * frame, int flags )
-{
- LIST * targets = lol_get( frame->args, 0 );
- LISTITER iter = list_begin( targets );
- LISTITER const end = list_end( targets );
- for ( ; iter != end; iter = list_next( iter ) )
- bindtarget( list_item( iter ) )->flags |= T_FLAG_PRECIOUS;
- return L0;
-}
-
-
-LIST * builtin_self_path( FRAME * frame, int flags )
-{
- extern char const * saved_argv0;
- char * p = executable_path( saved_argv0 );
- if ( p )
- {
- LIST * const result = list_new( object_new( p ) );
- free( p );
- return result;
- }
- return L0;
-}
-
-
-LIST * builtin_makedir( FRAME * frame, int flags )
-{
- LIST * const path = lol_get( frame->args, 0 );
- return file_mkdir( object_str( list_front( path ) ) )
- ? L0
- : list_new( object_copy( list_front( path ) ) );
-}
-
-
-#ifdef HAVE_PYTHON
-
-LIST * builtin_python_import_rule( FRAME * frame, int flags )
-{
- static int first_time = 1;
- char const * python_module = object_str( list_front( lol_get( frame->args,
- 0 ) ) );
- char const * python_function = object_str( list_front( lol_get( frame->args,
- 1 ) ) );
- OBJECT * jam_module = list_front( lol_get( frame->args, 2 ) );
- OBJECT * jam_rule = list_front( lol_get( frame->args, 3 ) );
-
- PyObject * pName;
- PyObject * pModule;
- PyObject * pDict;
- PyObject * pFunc;
-
- if ( first_time )
- {
- /* At the first invocation, we add the value of the global
- * EXTRA_PYTHONPATH to the sys.path Python variable.
- */
- LIST * extra = 0;
- module_t * outer_module = frame->module;
- LISTITER iter, end;
-
- first_time = 0;
-
- extra = var_get( root_module(), constant_extra_pythonpath );
-
- iter = list_begin( extra ), end = list_end( extra );
- for ( ; iter != end; iter = list_next( iter ) )
- {
- string buf[ 1 ];
- string_new( buf );
- string_append( buf, "import sys\nsys.path.append(\"" );
- string_append( buf, object_str( list_item( iter ) ) );
- string_append( buf, "\")\n" );
- PyRun_SimpleString( buf->value );
- string_free( buf );
- }
- }
-
- pName = PyString_FromString( python_module );
- pModule = PyImport_Import( pName );
- Py_DECREF( pName );
-
- if ( pModule != NULL )
- {
- pDict = PyModule_GetDict( pModule );
- pFunc = PyDict_GetItemString( pDict, python_function );
-
- if ( pFunc && PyCallable_Check( pFunc ) )
- {
- module_t * m = bindmodule( jam_module );
- new_rule_body( m, jam_rule, function_python( pFunc, 0 ), 0 );
- }
- else
- {
- if ( PyErr_Occurred() )
- PyErr_Print();
- fprintf( stderr, "Cannot find function \"%s\"\n", python_function );
- }
- Py_DECREF( pModule );
- }
- else
- {
- PyErr_Print();
- fprintf( stderr, "Failed to load \"%s\"\n", python_module );
- }
- return L0;
-
-}
-
-#endif /* #ifdef HAVE_PYTHON */
-
-
-void lol_build( LOL * lol, char const * * elements )
-{
- LIST * l = L0;
- lol_init( lol );
-
- while ( elements && *elements )
- {
- if ( !strcmp( *elements, ":" ) )
- {
- lol_add( lol, l );
- l = L0;
- }
- else
- {
- l = list_push_back( l, object_new( *elements ) );
- }
- ++elements;
- }
-
- if ( l != L0 )
- lol_add( lol, l );
-}
-
-
-#ifdef HAVE_PYTHON
-
-/*
- * Calls the bjam rule specified by name passed in 'args'. The name is looked up
- * in the context of bjam's 'python_interface' module. Returns the list of
- * strings returned by the rule.
- */
-
-PyObject * bjam_call( PyObject * self, PyObject * args )
-{
- FRAME inner[ 1 ];
- LIST * result;
- PARSE * p;
- OBJECT * rulename;
-
- /* Build up the list of arg lists. */
- frame_init( inner );
- inner->prev = 0;
- inner->prev_user = 0;
- inner->module = bindmodule( constant_python_interface );
-
- /* Extract the rule name and arguments from 'args'. */
-
- /* PyTuple_GetItem returns borrowed reference. */
- rulename = object_new( PyString_AsString( PyTuple_GetItem( args, 0 ) ) );
- {
- int i = 1;
- int size = PyTuple_Size( args );
- for ( ; i < size; ++i )
- {
- PyObject * a = PyTuple_GetItem( args, i );
- if ( PyString_Check( a ) )
- {
- lol_add( inner->args, list_new( object_new(
- PyString_AsString( a ) ) ) );
- }
- else if ( PySequence_Check( a ) )
- {
- LIST * l = 0;
- int s = PySequence_Size( a );
- int i = 0;
- for ( ; i < s; ++i )
- {
- /* PySequence_GetItem returns new reference. */
- PyObject * e = PySequence_GetItem( a, i );
- char * s = PyString_AsString( e );
- if ( !s )
- {
- printf( "Invalid parameter type passed from Python\n" );
- exit( 1 );
- }
- l = list_push_back( l, object_new( s ) );
- Py_DECREF( e );
- }
- lol_add( inner->args, l );
- }
- }
- }
-
- result = evaluate_rule( bindrule( rulename, inner->module), rulename, inner );
- object_free( rulename );
-
- frame_free( inner );
-
- /* Convert the bjam list into a Python list result. */
- {
- PyObject * const pyResult = PyList_New( list_length( result ) );
- int i = 0;
- LISTITER iter = list_begin( result );
- LISTITER const end = list_end( result );
- for ( ; iter != end; iter = list_next( iter ) )
- {
- PyList_SetItem( pyResult, i, PyString_FromString( object_str(
- list_item( iter ) ) ) );
- i += 1;
- }
- list_free( result );
- return pyResult;
- }
-}
-
-
-/*
- * Accepts four arguments:
- * - module name
- * - rule name,
- * - Python callable.
- * - (optional) bjam language function signature.
- * Creates a bjam rule with the specified name in the specified module, which
- * will invoke the Python callable.
- */
-
-PyObject * bjam_import_rule( PyObject * self, PyObject * args )
-{
- char * module;
- char * rule;
- PyObject * func;
- PyObject * bjam_signature = NULL;
- module_t * m;
- RULE * r;
- OBJECT * module_name;
- OBJECT * rule_name;
-
- if ( !PyArg_ParseTuple( args, "ssO|O:import_rule",
- &module, &rule, &func, &bjam_signature ) )
- return NULL;
-
- if ( !PyCallable_Check( func ) )
- {
- PyErr_SetString( PyExc_RuntimeError, "Non-callable object passed to "
- "bjam.import_rule" );
- return NULL;
- }
-
- module_name = *module ? object_new( module ) : 0;
- m = bindmodule( module_name );
- if ( module_name )
- object_free( module_name );
- rule_name = object_new( rule );
- new_rule_body( m, rule_name, function_python( func, bjam_signature ), 0 );
- object_free( rule_name );
-
- Py_INCREF( Py_None );
- return Py_None;
-}
-
-
-/*
- * Accepts four arguments:
- * - an action name
- * - an action body
- * - a list of variable that will be bound inside the action
- * - integer flags.
- * Defines an action on bjam side.
- */
-
-PyObject * bjam_define_action( PyObject * self, PyObject * args )
-{
- char * name;
- char * body;
- module_t * m;
- PyObject * bindlist_python;
- int flags;
- LIST * bindlist = L0;
- int n;
- int i;
- OBJECT * name_str;
- FUNCTION * body_func;
-
- if ( !PyArg_ParseTuple( args, "ssO!i:define_action", &name, &body,
- &PyList_Type, &bindlist_python, &flags ) )
- return NULL;
-
- n = PyList_Size( bindlist_python );
- for ( i = 0; i < n; ++i )
- {
- PyObject * next = PyList_GetItem( bindlist_python, i );
- if ( !PyString_Check( next ) )
- {
- PyErr_SetString( PyExc_RuntimeError, "bind list has non-string "
- "type" );
- return NULL;
- }
- bindlist = list_push_back( bindlist, object_new( PyString_AsString( next
- ) ) );
- }
-
- name_str = object_new( name );
- body_func = function_compile_actions( body, constant_builtin, -1 );
- new_rule_actions( root_module(), name_str, body_func, bindlist, flags );
- function_free( body_func );
- object_free( name_str );
-
- Py_INCREF( Py_None );
- return Py_None;
-}
-
-
-/*
- * Returns the value of a variable in root Jam module.
- */
-
-PyObject * bjam_variable( PyObject * self, PyObject * args )
-{
- char * name;
- LIST * value;
- PyObject * result;
- int i;
- OBJECT * varname;
- LISTITER iter;
- LISTITER end;
-
- if ( !PyArg_ParseTuple( args, "s", &name ) )
- return NULL;
-
- varname = object_new( name );
- value = var_get( root_module(), varname );
- object_free( varname );
- iter = list_begin( value );
- end = list_end( value );
-
- result = PyList_New( list_length( value ) );
- for ( i = 0; iter != end; iter = list_next( iter ), ++i )
- PyList_SetItem( result, i, PyString_FromString( object_str( list_item(
- iter ) ) ) );
-
- return result;
-}
-
-
-PyObject * bjam_backtrace( PyObject * self, PyObject * args )
-{
- PyObject * result = PyList_New( 0 );
- struct frame * f = frame_before_python_call;
-
- for ( ; f = f->prev; )
- {
- PyObject * tuple = PyTuple_New( 4 );
- char const * file;
- int line;
- char buf[ 32 ];
- string module_name[ 1 ];
-
- get_source_line( f, &file, &line );
- sprintf( buf, "%d", line );
- string_new( module_name );
- if ( f->module->name )
- {
- string_append( module_name, object_str( f->module->name ) );
- string_append( module_name, "." );
- }
-
- /* PyTuple_SetItem steals reference. */
- PyTuple_SetItem( tuple, 0, PyString_FromString( file ) );
- PyTuple_SetItem( tuple, 1, PyString_FromString( buf ) );
- PyTuple_SetItem( tuple, 2, PyString_FromString( module_name->value ) );
- PyTuple_SetItem( tuple, 3, PyString_FromString( f->rulename ) );
-
- string_free( module_name );
-
- PyList_Append( result, tuple );
- Py_DECREF( tuple );
- }
- return result;
-}
-
-PyObject * bjam_caller( PyObject * self, PyObject * args )
-{
- return PyString_FromString( frame_before_python_call->prev->module->name ?
- object_str( frame_before_python_call->prev->module->name ) : "" );
-}
-
-#endif /* #ifdef HAVE_PYTHON */
-
-
-#ifdef HAVE_POPEN
-
-#if defined(_MSC_VER) || defined(__BORLANDC__)
- #define popen windows_popen_wrapper
- #define pclose _pclose
-
- /*
- * This wrapper is a workaround for a funny _popen() feature on Windows
- * where it eats external quotes in some cases. The bug seems to be related
- * to the quote stripping functionality used by the Windows cmd.exe
- * interpreter when its /S is not specified.
- *
- * Cleaned up quote from the cmd.exe help screen as displayed on Windows XP
- * SP3:
- *
- * 1. If all of the following conditions are met, then quote characters on
- * the command line are preserved:
- *
- * - no /S switch
- * - exactly two quote characters
- * - no special characters between the two quote characters, where
- * special is one of: &<>()@^|
- * - there are one or more whitespace characters between the two quote
- * characters
- * - the string between the two quote characters is the name of an
- * executable file.
- *
- * 2. Otherwise, old behavior is to see if the first character is a quote
- * character and if so, strip the leading character and remove the last
- * quote character on the command line, preserving any text after the
- * last quote character.
- *
- * This causes some commands containing quotes not to be executed correctly.
- * For example:
- *
- * "\Long folder name\aaa.exe" --name="Jurko" --no-surname
- *
- * would get its outermost quotes stripped and would be executed as:
- *
- * \Long folder name\aaa.exe" --name="Jurko --no-surname
- *
- * which would report an error about '\Long' not being a valid command.
- *
- * cmd.exe help seems to indicate it would be enough to add an extra space
- * character in front of the command to avoid this but this does not work,
- * most likely due to the shell first stripping all leading whitespace
- * characters from the command.
- *
- * Solution implemented here is to quote the whole command in case it
- * contains any quote characters. Note thought this will not work correctly
- * should Windows ever 'fix' this feature.
- * (03.06.2008.) (Jurko)
- */
- static FILE * windows_popen_wrapper( char const * command,
- char const * mode )
- {
- int const extra_command_quotes_needed = !!strchr( command, '"' );
- string quoted_command;
- FILE * result;
-
- if ( extra_command_quotes_needed )
- {
- string_new( "ed_command );
- string_append( "ed_command, "\"" );
- string_append( "ed_command, command );
- string_append( "ed_command, "\"" );
- command = quoted_command.value;
- }
-
- result = _popen( command, "r" );
-
- if ( extra_command_quotes_needed )
- string_free( "ed_command );
-
- return result;
- }
-#endif /* defined(_MSC_VER) || defined(__BORLANDC__) */
-
-
-static char * rtrim( char * const s )
-{
- char * p = s;
- while ( *p ) ++p;
- for ( --p; p >= s && isspace( *p ); *p-- = 0 );
- return s;
-}
-
-
-LIST * builtin_shell( FRAME * frame, int flags )
-{
- LIST * command = lol_get( frame->args, 0 );
- LIST * result = L0;
- string s;
- int ret;
- char buffer[ 1024 ];
- FILE * p = NULL;
- int exit_status = -1;
- int exit_status_opt = 0;
- int no_output_opt = 0;
- int strip_eol_opt = 0;
-
- /* Process the variable args options. */
- {
- int a = 1;
- LIST * arg = lol_get( frame->args, a );
- for ( ; !list_empty( arg ); arg = lol_get( frame->args, ++a ) )
- {
- if ( !strcmp( "exit-status", object_str( list_front( arg ) ) ) )
- exit_status_opt = 1;
- else if ( !strcmp( "no-output", object_str( list_front( arg ) ) ) )
- no_output_opt = 1;
- else if ( !strcmp("strip-eol", object_str( list_front( arg ) ) ) )
- strip_eol_opt = 1;
- }
- }
-
- /* The following fflush() call seems to be indicated as a workaround for a
- * popen() bug on POSIX implementations related to synhronizing input
- * stream positions for the called and the calling process.
- */
- fflush( NULL );
-
- p = popen( object_str( list_front( command ) ), "r" );
- if ( p == NULL )
- return L0;
-
- string_new( &s );
-
- while ( ( ret = fread( buffer, sizeof( char ), sizeof( buffer ) - 1, p ) ) >
- 0 )
- {
- buffer[ ret ] = 0;
- if ( !no_output_opt )
- {
- if ( strip_eol_opt )
- rtrim( buffer );
- string_append( &s, buffer );
- }
- }
-
- exit_status = pclose( p );
-
- /* The command output is returned first. */
- result = list_new( object_new( s.value ) );
- string_free( &s );
-
- /* The command exit result next. */
- if ( exit_status_opt )
- {
- if ( WIFEXITED( exit_status ) )
- exit_status = WEXITSTATUS( exit_status );
- else
- exit_status = -1;
- sprintf( buffer, "%d", exit_status );
- result = list_push_back( result, object_new( buffer ) );
- }
-
- return result;
-}
-
-#else /* #ifdef HAVE_POPEN */
-
-LIST * builtin_shell( FRAME * frame, int flags )
-{
- return L0;
-}
-
-#endif /* #ifdef HAVE_POPEN */
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/builtins.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/builtins.h b/ext/kenlm/jam-files/engine/builtins.h
deleted file mode 100644
index b7a967c..0000000
--- a/ext/kenlm/jam-files/engine/builtins.h
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright 1993-2002 Christopher Seiwald and Perforce Software, Inc.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-#ifndef JAM_BUILTINS_H
-# define JAM_BUILTINS_H
-
-# include "frames.h"
-
-/*
- * builtins.h - compile parsed jam statements
- */
-
-void load_builtins();
-void init_set();
-void init_path();
-void init_regex();
-void init_property_set();
-void init_sequence();
-void init_order();
-
-void property_set_done();
-
-LIST *builtin_calc( FRAME * frame, int flags );
-LIST *builtin_depends( FRAME * frame, int flags );
-LIST *builtin_rebuilds( FRAME * frame, int flags );
-LIST *builtin_echo( FRAME * frame, int flags );
-LIST *builtin_exit( FRAME * frame, int flags );
-LIST *builtin_flags( FRAME * frame, int flags );
-LIST *builtin_glob( FRAME * frame, int flags );
-LIST *builtin_glob_recursive( FRAME * frame, int flags );
-LIST *builtin_subst( FRAME * frame, int flags );
-LIST *builtin_match( FRAME * frame, int flags );
-LIST *builtin_split_by_characters( FRAME * frame, int flags );
-LIST *builtin_hdrmacro( FRAME * frame, int flags );
-LIST *builtin_rulenames( FRAME * frame, int flags );
-LIST *builtin_varnames( FRAME * frame, int flags );
-LIST *builtin_delete_module( FRAME * frame, int flags );
-LIST *builtin_import( FRAME * frame, int flags );
-LIST *builtin_export( FRAME * frame, int flags );
-LIST *builtin_caller_module( FRAME * frame, int flags );
-LIST *builtin_backtrace( FRAME * frame, int flags );
-LIST *builtin_pwd( FRAME * frame, int flags );
-LIST *builtin_update( FRAME * frame, int flags );
-LIST *builtin_update_now( FRAME * frame, int flags );
-LIST *builtin_import_module( FRAME * frame, int flags );
-LIST *builtin_imported_modules( FRAME * frame, int flags );
-LIST *builtin_instance( FRAME * frame, int flags );
-LIST *builtin_sort( FRAME * frame, int flags );
-LIST *builtin_normalize_path( FRAME * frame, int flags );
-LIST *builtin_native_rule( FRAME * frame, int flags );
-LIST *builtin_has_native_rule( FRAME * frame, int flags );
-LIST *builtin_user_module( FRAME * frame, int flags );
-LIST *builtin_nearest_user_location( FRAME * frame, int flags );
-LIST *builtin_check_if_file( FRAME * frame, int flags );
-LIST *builtin_python_import_rule( FRAME * frame, int flags );
-LIST *builtin_shell( FRAME * frame, int flags );
-LIST *builtin_md5( FRAME * frame, int flags );
-LIST *builtin_file_open( FRAME * frame, int flags );
-LIST *builtin_pad( FRAME * frame, int flags );
-LIST *builtin_precious( FRAME * frame, int flags );
-LIST *builtin_self_path( FRAME * frame, int flags );
-LIST *builtin_makedir( FRAME * frame, int flags );
-
-void backtrace( FRAME *frame );
-extern int last_update_now_status;
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/bump_version.py
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/bump_version.py b/ext/kenlm/jam-files/engine/bump_version.py
deleted file mode 100644
index 1771422..0000000
--- a/ext/kenlm/jam-files/engine/bump_version.py
+++ /dev/null
@@ -1,98 +0,0 @@
-#!/usr/bin/python
-
-# This script is used to bump the bjam version. It takes a single argument, e.g
-#
-# ./bump_version.py 3.1.9
-#
-# and updates all the necessary files.
-#
-# Copyright 2006 Rene Rivera.
-# Copyright 2005-2006 Vladimir Prus.
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-
-import os
-import os.path
-import re
-import string
-import sys
-
-srcdir = os.path.abspath(os.path.dirname(__file__))
-docdir = os.path.abspath(os.path.join(srcdir, "..", "doc"))
-
-
-def edit(file, *replacements):
- print(" '%s'..." % file)
- f = open(file, 'r')
- text = f.read()
- f.close()
- for (source, target) in replacements:
- text, n = re.compile(source, re.MULTILINE).subn(target, text)
- assert n > 0
- f = open(file, 'w')
- f.write(text)
- f.close()
-
-
-def make_edits(ver):
- ver03 = (list(ver) + [0] * 3)[0:3]
- ver02 = ver03[0:2]
-
- join = lambda v, s : s.join(str(x) for x in v)
- dotJoin = lambda v : join(v, ".")
-
- print("Setting version to %s" % str(ver03))
-
- edit(os.path.join(srcdir, "boost-jam.spec"),
- ('^(Version:) .*$', '\\1 %s' % dotJoin(ver03)))
-
- edit(os.path.join(srcdir, "build.jam"),
- ('^(_VERSION_ =).* ;$', '\\1 %s ;' % join(ver03, " ")))
-
- edit(os.path.join(docdir, "bjam.qbk"),
- ('(\[version).*(\])', '\\1: %s\\2' % dotJoin(ver03)),
- ('(\[def :version:).*(\])', '\\1 %s\\2' % dotJoin(ver03)))
-
- edit(os.path.join(srcdir, "patchlevel.h"),
- ('^(#define VERSION_MAJOR) .*$', '\\1 %s' % ver03[0]),
- ('^(#define VERSION_MINOR) .*$', '\\1 %s' % ver03[1]),
- ('^(#define VERSION_PATCH) .*$', '\\1 %s' % ver03[2]),
- ('^(#define VERSION_MAJOR_SYM) .*$', '\\1 "%02d"' % ver03[0]),
- ('^(#define VERSION_MINOR_SYM) .*$', '\\1 "%02d"' % ver03[1]),
- ('^(#define VERSION_PATCH_SYM) .*$', '\\1 "%02d"' % ver03[2]),
- ('^(#define VERSION) .*$', '\\1 "%s"' % dotJoin(ver)),
- ('^(#define JAMVERSYM) .*$', '\\1 "JAMVERSION=%s"' % dotJoin(ver02)))
-
-
-def main():
- if len(sys.argv) < 2:
- print("Expect new version as argument.")
- sys.exit(1)
- if len(sys.argv) > 3:
- print("Too many arguments.")
- sys.exit(1)
-
- version = sys.argv[1].split(".")
- if len(version) > 3:
- print("Expect version argument in the format: <MAJOR>.<MINOR>.<PATCH>")
- sys.exit(1)
-
- try:
- version = list(int(x) for x in version)
- except ValueError:
- print("Version values must be valid integers.")
- sys.exit(1)
-
- while version and version[-1] == 0:
- version.pop()
-
- if not version:
- print("At least one of the version values must be positive.")
- sys.exit()
-
- make_edits(version)
-
-
-if __name__ == '__main__':
- main()
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/class.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/class.c b/ext/kenlm/jam-files/engine/class.c
deleted file mode 100644
index a4abfaa..0000000
--- a/ext/kenlm/jam-files/engine/class.c
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Copyright Vladimir Prus 2003.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#include "class.h"
-
-#include "constants.h"
-#include "frames.h"
-#include "hash.h"
-#include "lists.h"
-#include "object.h"
-#include "rules.h"
-#include "strings.h"
-#include "variable.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-
-
-static struct hash * classes = 0;
-
-
-static void check_defined( LIST * class_names )
-{
- LISTITER iter = list_begin( class_names );
- LISTITER const end = list_end( class_names );
- for ( ; iter != end; iter = list_next( iter ) )
- {
- if ( !hash_find( classes, list_item( iter ) ) )
- {
- printf( "Class %s is not defined\n", object_str( list_item( iter ) )
- );
- abort();
- }
- }
-}
-
-
-static OBJECT * class_module_name( OBJECT * declared_name )
-{
- string name[ 1 ];
- OBJECT * result;
-
- string_new( name );
- string_append( name, "class@" );
- string_append( name, object_str( declared_name ) );
-
- result = object_new( name->value );
- string_free( name );
-
- return result;
-}
-
-
-struct import_base_data
-{
- OBJECT * base_name;
- module_t * base_module;
- module_t * class_module;
-};
-
-
-static void import_base_rule( void * r_, void * d_ )
-{
- RULE * r = (RULE *)r_;
- RULE * ir1;
- RULE * ir2;
- struct import_base_data * d = (struct import_base_data *)d_;
- OBJECT * qname;
-
- string qualified_name[ 1 ];
- string_new ( qualified_name );
- string_append ( qualified_name, object_str( d->base_name ) );
- string_push_back( qualified_name, '.' );
- string_append ( qualified_name, object_str( r->name ) );
- qname = object_new( qualified_name->value );
- string_free( qualified_name );
-
- ir1 = import_rule( r, d->class_module, r->name );
- ir2 = import_rule( r, d->class_module, qname );
-
- object_free( qname );
-
- /* Copy 'exported' flag. */
- ir1->exported = ir2->exported = r->exported;
-
- /* If we are importing a class method, localize it. */
- if ( ( r->module == d->base_module ) || ( r->module->class_module &&
- ( r->module->class_module == d->base_module ) ) )
- {
- rule_localize( ir1, d->class_module );
- rule_localize( ir2, d->class_module );
- }
-}
-
-
-/*
- * For each exported rule 'n', declared in class module for base, imports that
- * rule in 'class' as 'n' and as 'base.n'. Imported rules are localized and
- * marked as exported.
- */
-
-static void import_base_rules( module_t * class_, OBJECT * base )
-{
- OBJECT * module_name = class_module_name( base );
- module_t * base_module = bindmodule( module_name );
- LIST * imported;
- struct import_base_data d;
- d.base_name = base;
- d.base_module = base_module;
- d.class_module = class_;
- object_free( module_name );
-
- if ( base_module->rules )
- hashenumerate( base_module->rules, import_base_rule, &d );
-
- imported = imported_modules( base_module );
- import_module( imported, class_ );
- list_free( imported );
-}
-
-
-OBJECT * make_class_module( LIST * xname, LIST * bases, FRAME * frame )
-{
- OBJECT * name = class_module_name( list_front( xname ) );
- OBJECT * * pp;
- module_t * class_module = 0;
- module_t * outer_module = frame->module;
- int found;
-
- if ( !classes )
- classes = hashinit( sizeof( OBJECT * ), "classes" );
-
- pp = (OBJECT * *)hash_insert( classes, list_front( xname ), &found );
- if ( !found )
- {
- *pp = object_copy( list_front( xname ) );
- }
- else
- {
- printf( "Class %s already defined\n", object_str( list_front( xname ) )
- );
- abort();
- }
- check_defined( bases );
-
- class_module = bindmodule( name );
-
- {
- /*
- Initialize variables that Boost.Build inserts in every object.
- We want to avoid creating the object's hash if it isn't needed.
- */
- int num = class_module->num_fixed_variables;
- module_add_fixed_var( class_module, constant_name, &num );
- module_add_fixed_var( class_module, constant_class, &num );
- module_set_fixed_variables( class_module, num );
- }
-
- var_set( class_module, constant_name, xname, VAR_SET );
- var_set( class_module, constant_bases, bases, VAR_SET );
-
- {
- LISTITER iter = list_begin( bases );
- LISTITER const end = list_end( bases );
- for ( ; iter != end; iter = list_next( iter ) )
- import_base_rules( class_module, list_item( iter ) );
- }
-
- return name;
-}
-
-
-static void free_class( void * xclass, void * data )
-{
- object_free( *(OBJECT * *)xclass );
-}
-
-
-void class_done( void )
-{
- if ( classes )
- {
- hashenumerate( classes, free_class, (void *)0 );
- hashdone( classes );
- classes = 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/class.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/class.h b/ext/kenlm/jam-files/engine/class.h
deleted file mode 100644
index 256d298..0000000
--- a/ext/kenlm/jam-files/engine/class.h
+++ /dev/null
@@ -1,14 +0,0 @@
-/* Copyright Vladimir Prus 2003. Distributed under the Boost */
-/* Software License, Version 1.0. (See accompanying */
-/* file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) */
-
-#ifndef CLASS_H_VP_2003_08_01
-#define CLASS_H_VP_2003_08_01
-
-#include "lists.h"
-#include "frames.h"
-
-OBJECT * make_class_module( LIST * xname, LIST * bases, FRAME * frame );
-void class_done( void );
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/command.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/command.c b/ext/kenlm/jam-files/engine/command.c
deleted file mode 100644
index 571b8b6..0000000
--- a/ext/kenlm/jam-files/engine/command.c
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Copyright 1993, 1995 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * command.c - maintain lists of commands
- */
-
-#include "jam.h"
-#include "command.h"
-
-#include "lists.h"
-#include "rules.h"
-
-#include <assert.h>
-
-
-/*
- * cmd_new() - return a new CMD.
- */
-
-CMD * cmd_new( RULE * rule, LIST * targets, LIST * sources, LIST * shell )
-{
- CMD * cmd = (CMD *)BJAM_MALLOC( sizeof( CMD ) );
- FRAME frame[ 1 ];
-
- assert( cmd );
- cmd->rule = rule;
- cmd->shell = shell;
- cmd->next = 0;
- cmd->noop = 0;
-
- lol_init( &cmd->args );
- lol_add( &cmd->args, targets );
- lol_add( &cmd->args, sources );
- string_new( cmd->buf );
-
- frame_init( frame );
- frame->module = rule->module;
- lol_init( frame->args );
- lol_add( frame->args, list_copy( targets ) );
- lol_add( frame->args, list_copy( sources ) );
- function_run_actions( rule->actions->command, frame, stack_global(),
- cmd->buf );
- frame_free( frame );
-
- return cmd;
-}
-
-
-/*
- * cmd_free() - free a CMD
- */
-
-void cmd_free( CMD * cmd )
-{
- lol_free( &cmd->args );
- list_free( cmd->shell );
- string_free( cmd->buf );
- BJAM_FREE( (void *)cmd );
-}
-
-
-/*
- * cmd_release_targets_and_shell()
- *
- * Makes the CMD release its hold on its targets & shell lists and forget
- * about them. Useful in case caller still has references to those lists and
- * wants to reuse them after freeing the CMD object.
- */
-
-void cmd_release_targets_and_shell( CMD * cmd )
-{
- cmd->args.list[ 0 ] = L0; /* targets */
- cmd->shell = L0; /* shell */
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/command.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/command.h b/ext/kenlm/jam-files/engine/command.h
deleted file mode 100644
index ba65577..0000000
--- a/ext/kenlm/jam-files/engine/command.h
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Copyright 1994 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/*
- * command.h - the CMD structure and routines to manipulate them
- *
- * Both ACTION and CMD contain a rule, targets, and sources. An
- * ACTION describes a rule to be applied to the given targets and
- * sources; a CMD is what actually gets executed by the shell. The
- * differences are due to:
- *
- * ACTIONS must be combined if 'actions together' is given.
- * ACTIONS must be split if 'actions piecemeal' is given.
- * ACTIONS must have current sources omitted for 'actions updated'.
- *
- * The CMD datatype holds a single command that is to be executed
- * against a target, and they can chain together to represent the
- * full collection of commands used to update a target.
- *
- * Structures:
- *
- * CMD - an action, ready to be formatted into a buffer and executed.
- *
- * External routines:
- *
- * cmd_new() - return a new CMD or 0 if too many args.
- * cmd_free() - delete CMD and its parts.
- * cmd_next() - walk the CMD chain.
- * cmd_release_targets_and_shell() - CMD forgets about its targets & shell.
- */
-
-
-/*
- * CMD - an action, ready to be formatted into a buffer and executed.
- */
-
-#ifndef COMMAND_SW20111118_H
-#define COMMAND_SW20111118_H
-
-#include "lists.h"
-#include "rules.h"
-#include "strings.h"
-
-
-typedef struct _cmd CMD;
-struct _cmd
-{
- CMD * next;
- RULE * rule; /* rule->actions contains shell script */
- LIST * shell; /* $(JAMSHELL) value */
- LOL args; /* LISTs for $(<), $(>) */
- string buf[ 1 ]; /* actual commands */
- int noop; /* no-op commands should be faked instead of executed */
-};
-
-CMD * cmd_new
-(
- RULE * rule, /* rule (referenced) */
- LIST * targets, /* $(<) (ownership transferred) */
- LIST * sources, /* $(>) (ownership transferred) */
- LIST * shell /* $(JAMSHELL) (ownership transferred) */
-);
-
-void cmd_release_targets_and_shell( CMD * );
-
-void cmd_free( CMD * );
-
-#define cmd_next( c ) ((c)->next)
-
-#endif
[30/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/Jambase
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/Jambase b/ext/kenlm/jam-files/engine/Jambase
deleted file mode 100644
index 94f8fbd..0000000
--- a/ext/kenlm/jam-files/engine/Jambase
+++ /dev/null
@@ -1,2473 +0,0 @@
-#
-# /+\
-# +\ Copyright 1993, 2000 Christopher Seiwald.
-# \+/
-#
-# This file is part of Jam - see jam.c for Copyright information.
-#
-
-# This file is ALSO:
-# Copyright 2001-2004 David Abrahams.
-# Copyright 2002-2004 Rene Rivera.
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-if $(NT)
-{
- SLASH ?= \\ ;
-}
-SLASH ?= / ;
-
-
-# Glob for patterns in the directories starting from the given start directory,
-# up to and including the root of the file-system. We stop globbing as soon as
-# we find at least one match.
-#
-rule find-to-root ( dir : patterns + )
-{
- local globs = [ GLOB $(dir) : $(patterns) ] ;
- while ! $(globs) && $(dir:P) != $(dir)
- {
- dir = $(dir:P) ;
- globs = [ GLOB $(dir) : $(patterns) ] ;
- }
- return $(globs) ;
-}
-
-
-# This global will hold the location of the user's boost-build.jam file.
-.boost-build-file = ;
-
-# This global will hold the location of the build system bootstrap file.
-.bootstrap-file = ;
-
-# Remember the value of $(BOOST_BUILD_PATH) supplied to us by the user.
-BOOST_BUILD_PATH.user-value = $(BOOST_BUILD_PATH) ;
-
-# On Unix only, when BOOST_BUILD_PATH is not supplied by the user, set it to a
-# sensible default value. This allows Boost.Build to work without any
-# environment variables, which is good in itself and also required by the Debian
-# Policy.
-if ! $(BOOST_BUILD_PATH) && $(UNIX)
-{
- BOOST_BUILD_PATH = /usr/share/boost-build ;
-}
-
-
-rule _poke ( module-name ? : variables + : value * )
-{
- module $(<)
- {
- $(>) = $(3) ;
- }
-}
-
-
-# This rule can be invoked from an optional user's boost-build.jam file to both
-# indicate where to find the build system files, and to load them. The path
-# indicated is relative to the location of the boost-build.jam file.
-#
-rule boost-build ( dir ? )
-{
- if $(.bootstrap-file)
- {
- ECHO "Error: Illegal attempt to re-bootstrap the build system by invoking" ;
- ECHO ;
- ECHO " 'boost-build" $(dir) ";'" ;
- ECHO ;
- EXIT "Please consult the documentation at 'http://www.boost.org'." ;
- }
-
- # Add the given directory to the path so we can find the build system. If
- # dir is empty, has no effect.
- BOOST_BUILD_PATH = $(dir:R=$(.boost-build-file:D)) $(BOOST_BUILD_PATH) ;
-
- # We might have just modified the *global* value of BOOST_BUILD_PATH. The
- # code that loads the rest of Boost.Build, in particular the site-config.jam
- # and user-config.jam configuration files uses os.environ, so we need to
- # update the value there.
- _poke .ENVIRON : BOOST_BUILD_PATH : $(BOOST_BUILD_PATH) ;
-
- # Try to find the build system bootstrap file 'bootstrap.jam'.
- local bootstrap-file = [ GLOB $(BOOST_BUILD_PATH) : bootstrap.jam ] ;
- .bootstrap-file = $(bootstrap-file[1]) ;
-
- # There is no bootstrap.jam we can find, exit with an error.
- if ! $(.bootstrap-file)
- {
- ECHO "Unable to load Boost.Build: could not find build system." ;
- ECHO --------------------------------------------------------- ;
- ECHO "$(.boost-build-file) attempted to load the build system by invoking" ;
- ECHO ;
- ECHO " 'boost-build" $(dir) ";'" ;
- ECHO ;
- ECHO "but we were unable to find \"bootstrap.jam\" in the specified directory" ;
- ECHO "or in BOOST_BUILD_PATH (searching "$(BOOST_BUILD_PATH:J=", ")")." ;
- ECHO ;
- EXIT "Please consult the documentation at 'http://www.boost.org'." ;
- }
-
- if [ MATCH .*(--debug-configuration).* : $(ARGV) ]
- {
- ECHO "notice: loading Boost.Build from"
- [ NORMALIZE_PATH $(.bootstrap-file:D) ] ;
- }
-
- # Load the build system, now that we know where to start from.
- include $(.bootstrap-file) ;
-}
-
-
-if [ MATCH .*(b2).* : $(ARGV[1]:BL) ]
- || [ MATCH .*(bjam).* : $(ARGV[1]:BL) ]
- || $(BOOST_ROOT) # A temporary measure so Jam works with Boost.Build v1.
-{
- # We attempt to load "boost-build.jam" by searching from the current
- # invocation directory up to the root of the file-system.
- #
- # boost-build.jam is expected to invoke the "boost-build" rule to load the
- # Boost.Build files.
-
- local search-path = $(BOOST_BUILD_PATH) $(BOOST_ROOT) ;
- local self = [ SELF_PATH ] ;
- local boost-build-relative = ../../share/boost-build ;
- local self-based-path = [ NORMALIZE_PATH $(boost-build-relative:R=$(self)) ] ;
-
- local boost-build-files =
- [ find-to-root [ PWD ] : boost-build.jam ]
- [ GLOB $(self-based-path) : boost-build.jam ]
- # Another temporary measure so Jam works with Boost.Build v1.
- [ GLOB $(search-path) : boost-build.jam ] ;
-
- .boost-build-file = $(boost-build-files[1]) ;
-
- # There is no boost-build.jam we can find, exit with an error, and
- # information.
- if ! $(.boost-build-file)
- {
- ECHO "Unable to load Boost.Build: could not find \"boost-build.jam\"" ;
- ECHO --------------------------------------------------------------- ;
-
- if ! [ MATCH .*(bjam).* : $(ARGV[1]:BL) ]
- {
- ECHO "BOOST_ROOT must be set, either in the environment, or " ;
- ECHO "on the command-line with -sBOOST_ROOT=..., to the root" ;
- ECHO "of the boost installation." ;
- ECHO ;
- }
-
- ECHO "Attempted search from" [ PWD ] "up to the root" ;
- ECHO "at" $(self-based-path) ;
- ECHO "and in these directories from BOOST_BUILD_PATH and BOOST_ROOT: "$(search-path:J=", ")"." ;
- EXIT "Please consult the documentation at 'http://www.boost.org'." ;
- }
-
- if [ MATCH .*(--debug-configuration).* : $(ARGV) ]
- {
- ECHO "notice: found boost-build.jam at"
- [ NORMALIZE_PATH $(.boost-build-file) ] ;
- }
-
- # Now load the boost-build.jam to get the build system loaded. This
- # incidentaly loads the users jamfile and attempts to build targets.
- #
- # We also set it up so we can tell whether we are loading the new V2 system
- # or the the old V1 system.
- include $(.boost-build-file) ;
-
- # Check that, at minimum, the bootstrap file was found.
- if ! $(.bootstrap-file)
- {
- ECHO "Unable to load Boost.Build" ;
- ECHO -------------------------- ;
- ECHO "\"$(.boost-build-file)\" was found by searching from" [ PWD ] "up to the root" ;
- ECHO "and in these directories from BOOST_BUILD_PATH and BOOST_ROOT: "$(search-path:J=", ")"." ;
- ECHO ;
- ECHO "However, it failed to call the \"boost-build\" rule to indicate" ;
- ECHO "the location of the build system." ;
- ECHO ;
- EXIT "Please consult the documentation at 'http://www.boost.org'." ;
- }
-}
-else
-{
-
-#
-# JAMBASE - jam 2.3 ruleset providing make(1)-like functionality
-#
-# Supports UNIX, NT, and VMS.
-#
-# 12/27/93 (seiwald) - purturb library sources with SOURCE_GRIST
-# 04/18/94 (seiwald) - use '?=' when setting OS specific vars
-# 04/21/94 (seiwald) - do RmTemps together
-# 05/05/94 (seiwald) - all supported C compilers support -o: relegate
-# RELOCATE as an option; set Ranlib to "" to disable it
-# 06/01/94 (seiwald) - new 'actions existing' to do existing sources
-# 08/25/94 (seiwald) - new ObjectCcFlags rule to append to per-target CCFLAGS
-# 08/29/94 (seiwald) - new ObjectHdrs rule to append to per-target HDRS
-# 09/19/94 (seiwald) - LinkLibraries and Undefs now append
-# - Rule names downshifted.
-# 10/06/94 (seiwald) - Dumb yyacc stuff moved into Jamfile.
-# 10/14/94 (seiwald) - (Crude) support for .s, .C, .cc, .cpp, and .f files.
-# 01/08/95 (seiwald) - Shell now handled with awk, not sed
-# 01/09/95 (seiwald) - Install* now take dest directory as target
-# 01/10/95 (seiwald) - All entries sorted.
-# 01/10/95 (seiwald) - NT support moved in, with LauraW's help.
-# 01/10/95 (seiwald) - VMS support moved in.
-# 02/06/95 (seiwald) - ObjectC++Flags and SubDirC++Flags added.
-# 02/07/95 (seiwald) - Iron out when HDRSEARCH uses "" or SEARCH_SOURCE.
-# 02/08/95 (seiwald) - SubDir works on VMS.
-# 02/14/95 (seiwald) - MkDir and entourage.
-# 04/30/95 (seiwald) - Use install -c flag so that it copies, not moves.
-# 07/10/95 (taylor) - Support for Microsoft C++.
-# 11/21/96 (peterk) - Support for BeOS
-# 07/19/99 (sickel) - Support for Mac OS X Server (and maybe client)
-# 02/18/00 (belmonte)- Support for Cygwin.
-
-# Special targets defined in this file:
-#
-# all - parent of first, shell, files, lib, exe
-# first - first dependency of 'all', for potential initialization
-# shell - parent of all Shell targets
-# files - parent of all File targets
-# lib - parent of all Library targets
-# exe - parent of all Main targets
-# dirs - parent of all MkDir targets
-# clean - removes all Shell, File, Library, and Main targets
-# uninstall - removes all Install targets
-#
-
-# Rules defined by this file:
-#
-# as obj.o : source.s ; .s -> .o
-# Bulk dir : files ; populate directory with many files
-# Cc obj.o : source.c ; .c -> .o
-# C++ obj.o : source.cc ; .cc -> .o
-# Clean clean : sources ; remove sources with 'jam clean'
-# File dest : source ; copy file
-# Fortran obj.o : source.f ; .f -> .o
-# GenFile source.c : program args ; make custom file
-# Hardlink target : source ; make link from source to target
-# HdrRule source : headers ; handle #includes
-# InstallInto dir : sources ; install any files
-# InstallBin dir : sources ; install binaries
-# InstallLib dir : sources ; install files
-# InstallFile dir : sources ; install files
-# InstallMan dir : sources ; install man pages
-# InstallShell dir : sources ; install shell scripts
-# Lex source.c : source.l ; .l -> .c
-# Library lib : source ; archive library from compiled sources
-# LibraryFromObjects lib : objects ; archive library from objects
-# LinkLibraries images : libraries ; bag libraries onto Mains
-# Main image : source ; link executable from compiled sources
-# MainFromObjects image : objects ; link executable from objects
-# MkDir dir ; make a directory, if not there
-# Object object : source ; compile object from source
-# ObjectCcFlags source : flags ; add compiler flags for object
-# ObjectC++Flags source : flags ; add compiler flags for object
-# ObjectHdrs source : dirs ; add include directories for object
-# Objects sources ; compile sources
-# RmTemps target : sources ; remove temp sources after target made
-# Setuid images ; mark executables Setuid
-# SubDir TOP d1 d2 ... ; start a subdirectory Jamfile
-# SubDirCcFlags flags ; add compiler flags until next SubDir
-# SubDirC++Flags flags ; add compiler flags until next SubDir
-# SubDirHdrs dirs ; add include dirs until next SubDir
-# SubInclude TOP d1 d2 ... ; include a subdirectory Jamfile
-# Shell exe : source ; make a shell executable
-# Undefines images : symbols ; save undef's for linking
-# UserObject object : source ; handle unknown suffixes for Object
-# Yacc source.c : source.y ; .y -> .c
-#
-# Utility rules that have no side effects (not supported):
-#
-# FAppendSuffix f1 f2 ... : $(SUF) ; return $(<) with suffixes
-# FConcat value ... ; return contatenated values
-# FDirName d1 d2 ... ; return path from root to dir
-# FGrist d1 d2 ... ; return d1!d2!...
-# FGristFiles value ; return $(value:G=$(SOURCE_GRIST))
-# FGristSourceFiles value ; return $(value:G=$(SOURCE_GRIST))
-# FRelPath d1 : d2 ; return rel path from d1 to d2
-# FSubDir d1 d2 ... ; return path to root
-#
-
-
-# Brief review of the jam language:
-#
-# Statements:
-# rule RULE - statements to process a rule
-# actions RULE - system commands to carry out target update
-#
-# Modifiers on actions:
-# together - multiple instances of same rule on target get executed
-# once with their sources ($(>)) concatenated
-# updated - refers to updated sources ($(>)) only
-# ignore - ignore return status of command
-# quietly - don't trace its execution unless verbose
-# piecemeal - iterate command each time with a small subset of $(>)
-# existing - refers to currently existing sources ($(>)) only
-# bind vars - subject to binding before expanding in actions
-#
-# Special rules:
-# ALWAYS - always build a target
-# DEPENDS - builds the dependency graph
-# ECHO - blurt out targets on stdout
-# EXIT - blurt out targets and exit
-# INCLUDES - marks sources as headers for target (a codependency)
-# NOCARE - don't panic if the target can't be built
-# NOUPDATE - create the target if needed but never update it
-# NOTFILE - ignore the timestamp of the target (it's not a file)
-# TEMPORARY - target need not be present if sources haven't changed
-#
-# Special variables set by jam:
-# $(<) - targets of a rule (to the left of the :)
-# $(>) - sources of a rule (to the right of the :)
-# $(xxx) - true on xxx (UNIX, VMS, NT, OS2, MAC)
-# $(OS) - name of OS - varies wildly
-# $(JAMVERSION) - version number (2.3)
-#
-# Special variables used by jam:
-# SEARCH - where to find something (used during binding and actions)
-# LOCATE - where to plop something not found with SEARCH
-# HDRRULE - rule to call to handle include files
-# HDRSCAN - egrep regex to extract include files
-#
-# Special targets:
-# all - default if none given on command line
-#
-
-# Initialize variables
-#
-
-#
-# OS specific variable settings
-#
-if $(NT)
-{
- # the list of supported toolsets on Windows NT and Windows 95/98
- #
- local SUPPORTED_TOOLSETS = "BORLANDC" "VC7" "VISUALC" "VISUALC16" "INTELC" "WATCOM"
- "MINGW" "LCC" ;
-
- # this variable holds the current toolset
- #
- TOOLSET = "" ;
-
- # if the JAM_TOOLSET environment variable is defined, check that it is
- # one of our supported values
- #
- if $(JAM_TOOLSET)
- {
- local t ;
-
- for t in $(SUPPORTED_TOOLSETS)
- {
- $(t) = $($(t):J=" ") ; # reconstitute paths with spaces in them
- if $(t) = $(JAM_TOOLSET) { TOOLSET = $(t) ; }
- }
-
- if ! $(TOOLSET)
- {
- ECHO "The JAM_TOOLSET environment variable is defined but its value" ;
- ECHO "is invalid, please use one of the following:" ;
- ECHO ;
-
- for t in $(SUPPORTED_TOOLSETS) { ECHO " " $(t) ; }
- EXIT ;
- }
- }
-
- # if TOOLSET is empty, we'll try to detect the toolset from other
- # environment variables to remain backwards compatible with Jam 2.3
- #
- if ! $(TOOLSET)
- {
- if $(BCCROOT)
- {
- TOOLSET = BORLANDC ;
- BORLANDC = $(BCCROOT:J=" ") ;
- }
- else if $(MSVC)
- {
- TOOLSET = VISUALC16 ;
- VISUALC16 = $(MSVC:J=" ") ;
- }
- else if $(MSVCNT)
- {
- TOOLSET = VISUALC ;
- VISUALC = $(MSVCNT:J=" ") ;
- }
- else if $(MSVCDir)
- {
- TOOLSET = VISUALC ;
- VISUALC = $(MSVCDir:J=" ") ;
- }
- else if $(MINGW)
- {
- TOOLSET = MINGW ;
- }
- else
- {
- ECHO "Jam cannot be run because, either:" ;
- ECHO " a. You didn't set BOOST_ROOT to indicate the root of your" ;
- ECHO " Boost installation." ;
- ECHO " b. You are trying to use stock Jam but didn't indicate which" ;
- ECHO " compilation toolset to use. To do so, follow these simple" ;
- ECHO " instructions:" ;
- ECHO ;
- ECHO " - define one of the following environment variable, with the" ;
- ECHO " appropriate value according to this list:" ;
- ECHO ;
- ECHO " Variable Toolset Description" ;
- ECHO ;
- ECHO " BORLANDC Borland C++ BC++ install path" ;
- ECHO " VISUALC Microsoft Visual C++ VC++ install path" ;
- ECHO " VISUALC16 Microsoft Visual C++ 16 bit VC++ 16 bit install" ;
- ECHO " INTELC Intel C/C++ IC++ install path" ;
- ECHO " WATCOM Watcom C/C++ Watcom install path" ;
- ECHO " MINGW MinGW (gcc) MinGW install path" ;
- ECHO " LCC Win32-LCC LCC-Win32 install path" ;
- ECHO ;
- ECHO " - define the JAM_TOOLSET environment variable with the *name*" ;
- ECHO " of the toolset variable you want to use." ;
- ECHO ;
- ECHO " e.g.: set VISUALC=C:\\Visual6" ;
- ECHO " set JAM_TOOLSET=VISUALC" ;
- EXIT ;
- }
- }
-
- CP ?= copy ;
- RM ?= del /f/q ;
- SLASH ?= \\ ;
- SUFLIB ?= .lib ;
- SUFOBJ ?= .obj ;
- SUFEXE ?= .exe ;
-
- if $(TOOLSET) = BORLANDC
- {
- ECHO "Compiler is Borland C++" ;
-
- AR ?= tlib /C /P64 ;
- CC ?= bcc32 ;
- CCFLAGS ?= -q -y -d -v -w-par -w-ccc -w-rch -w-pro -w-aus ;
- C++ ?= bcc32 ;
- C++FLAGS ?= -q -y -d -v -w-par -w-ccc -w-rch -w-pro -w-aus -P ;
- LINK ?= $(CC) ;
- LINKFLAGS ?= $(CCFLAGS) ;
- STDLIBPATH ?= $(BORLANDC)\\lib ;
- STDHDRS ?= $(BORLANDC)\\include ;
- NOARSCAN ?= true ;
- }
- else if $(TOOLSET) = VISUALC16
- {
- ECHO "Compiler is Microsoft Visual C++ 16 bit" ;
-
- AR ?= lib /nologo ;
- CC ?= cl /nologo ;
- CCFLAGS ?= /D \"WIN\" ;
- C++ ?= $(CC) ;
- C++FLAGS ?= $(CCFLAGS) ;
- LINK ?= $(CC) ;
- LINKFLAGS ?= $(CCFLAGS) ;
- LINKLIBS ?=
- \"$(VISUALC16)\\lib\\mlibce.lib\"
- \"$(VISUALC16)\\lib\\oldnames.lib\"
- ;
- LINKLIBS ?= ;
- NOARSCAN ?= true ;
- OPTIM ?= "" ;
- STDHDRS ?= $(VISUALC16)\\include ;
- UNDEFFLAG ?= "/u _" ;
- }
- else if $(TOOLSET) = VISUALC
- {
- ECHO "Compiler is Microsoft Visual C++" ;
-
- AR ?= lib ;
- AS ?= masm386 ;
- CC ?= cl /nologo ;
- CCFLAGS ?= "" ;
- C++ ?= $(CC) ;
- C++FLAGS ?= $(CCFLAGS) ;
- LINK ?= link /nologo ;
- LINKFLAGS ?= "" ;
- LINKLIBS ?= \"$(VISUALC)\\lib\\advapi32.lib\"
- # $(VISUALC)\\lib\\libc.lib
- # $(VISUALC)\\lib\\oldnames.lib
- \"$(VISUALC)\\lib\\gdi32.lib\"
- \"$(VISUALC)\\lib\\user32.lib\"
- \"$(VISUALC)\\lib\\kernel32.lib\" ;
- OPTIM ?= "" ;
- STDHDRS ?= $(VISUALC)\\include ;
- UNDEFFLAG ?= "/u _" ;
- }
- else if $(TOOLSET) = VC7
- {
- ECHO "Compiler is Microsoft Visual C++ .NET" ;
-
- AR ?= lib ;
- AS ?= masm386 ;
- CC ?= cl /nologo ;
- CCFLAGS ?= "" ;
- C++ ?= $(CC) ;
- C++FLAGS ?= $(CCFLAGS) ;
- LINK ?= link /nologo ;
- LINKFLAGS ?= "" ;
- LINKLIBS ?= \"$(VISUALC)\\PlatformSDK\\lib\\advapi32.lib\"
- # $(VISUALC)\\lib\\libc.lib
- # $(VISUALC)\\lib\\oldnames.lib
- \"$(VISUALC)\\PlatformSDK\\lib\\gdi32.lib\"
- \"$(VISUALC)\\PlatformSDK\\lib\\user32.lib\"
- \"$(VISUALC)\\PlatformSDK\\lib\\kernel32.lib\" ;
- OPTIM ?= "" ;
- STDHDRS ?= \"$(VISUALC)\\include\"
- \"$(VISUALC)\\PlatformSDK\\include\" ;
- UNDEFFLAG ?= "/u _" ;
- }
- else if $(TOOLSET) = INTELC
- {
- ECHO "Compiler is Intel C/C++" ;
-
- if ! $(VISUALC)
- {
- ECHO "As a special exception, when using the Intel C++ compiler, you need" ;
- ECHO "to define the VISUALC environment variable to indicate the location" ;
- ECHO "of your Visual C++ installation. Aborting.." ;
- EXIT ;
- }
-
- AR ?= lib ;
- AS ?= masm386 ;
- CC ?= icl /nologo ;
- CCFLAGS ?= "" ;
- C++ ?= $(CC) ;
- C++FLAGS ?= $(CCFLAGS) ;
- LINK ?= link /nologo ;
- LINKFLAGS ?= "" ;
- LINKLIBS ?= $(VISUALC)\\lib\\advapi32.lib
- # $(VISUALC)\\lib\\libc.lib
- # $(VISUALC)\\lib\\oldnames.lib
- $(VISUALC)\\lib\\kernel32.lib
- ;
- OPTIM ?= "" ;
- STDHDRS ?= $(INTELC)\include $(VISUALC)\\include ;
- UNDEFFLAG ?= "/u _" ;
- }
- else if $(TOOLSET) = WATCOM
- {
- ECHO "Compiler is Watcom C/C++" ;
-
- AR ?= wlib ;
- CC ?= wcc386 ;
- CCFLAGS ?= /zq /DWIN32 /I$(WATCOM)\\h ; # zq=quiet
- C++ ?= wpp386 ;
- C++FLAGS ?= $(CCFLAGS) ;
- CP ?= copy ;
- DOT ?= . ;
- DOTDOT ?= .. ;
- LINK ?= wcl386 ;
- LINKFLAGS ?= /zq ; # zq=quiet
- LINKLIBS ?= ;
- MV ?= move ;
- NOARSCAN ?= true ;
- OPTIM ?= ;
- RM ?= del /f ;
- SLASH ?= \\ ;
- STDHDRS ?= $(WATCOM)\\h $(WATCOM)\\h\\nt ;
- SUFEXE ?= .exe ;
- SUFLIB ?= .lib ;
- SUFOBJ ?= .obj ;
- UNDEFFLAG ?= "/u _" ;
- }
- else if $(TOOLSET) = MINGW
- {
- ECHO "Compiler is GCC with Mingw" ;
-
- AR ?= ar -ru ;
- CC ?= gcc ;
- CCFLAGS ?= "" ;
- C++ ?= $(CC) ;
- C++FLAGS ?= $(CCFLAGS) ;
- LINK ?= $(CC) ;
- LINKFLAGS ?= "" ;
- LINKLIBS ?= "" ;
- OPTIM ?= ;
- SUFOBJ = .o ;
- SUFLIB = .a ;
- SLASH = / ;
-# NOARSCAN ?= true ;
- }
- else if $(TOOLSET) = LCC
- {
- ECHO "Compiler is Win32-LCC" ;
-
- AR ?= lcclib ;
- CC ?= lcc ;
- CCFLAGS ?= "" ;
- C++ ?= $(CC) ;
- C++FLAGS ?= $(CCFLAGS) ;
- LINK ?= lcclnk ;
- LINKFLAGS ?= "" ;
- LINKLIBS ?= "" ;
- OPTIM ?= ;
- NOARSCAN = true ;
- }
- else
- {
-#
-# XXX: We need better comments here !!
-#
- EXIT On NT, set BCCROOT, MSVCNT, MINGW or MSVC to the root of the
- Borland or Microsoft directories. ;
- }
-
-}
-else if $(OS2)
-{
- # the list of supported toolsets on Windows NT and Windows 95/98
- #
- local SUPPORTED_TOOLSETS = "EMX" "WATCOM" ;
-
- # this variable holds the current toolset
- #
- TOOLSET = "" ;
-
- # if the JAM_TOOLSET environment variable is defined, check that it is
- # one of our supported values
- #
- if $(JAM_TOOLSET)
- {
- local t ;
-
- for t in $(SUPPORTED_TOOLSETS)
- {
- $(t) = $($(t):J=" ") ; # reconstitute paths with spaces in them
- if $(t) = $(JAM_TOOLSET) { TOOLSET = $(t) ; }
- }
-
- if ! $(TOOLSET)
- {
- ECHO "The JAM_TOOLSET environment variable is defined but its value" ;
- ECHO "is invalid, please use one of the following:" ;
- ECHO ;
-
- for t in $(SUPPORTED_TOOLSETS) { ECHO " " $(t) ; }
- EXIT ;
- }
- }
-
- # if TOOLSET is empty, we'll try to detect the toolset from other
- # environment variables to remain backwards compatible with Jam 2.3
- #
- if ! $(TOOLSET)
- {
- if $(watcom)
- {
- WATCOM = $(watcom:J=" ") ;
- TOOLSET = WATCOM ;
- }
- else
- {
- ECHO "Jam cannot be run because you didn't indicate which compilation toolset" ;
- ECHO "to use. To do so, follow these simple instructions:" ;
- ECHO ;
- ECHO " - define one of the following environment variable, with the" ;
- ECHO " appropriate value according to this list:" ;
- ECHO ;
- ECHO " Variable Toolset Description" ;
- ECHO ;
- ECHO " WATCOM Watcom C/C++ Watcom install path" ;
- ECHO " EMX EMX (gcc) EMX install path" ;
- ECHO " VISUALAGE IBM Visual Age C/C++ VisualAge install path" ;
- ECHO ;
- ECHO " - define the JAM_TOOLSET environment variable with the *name*" ;
- ECHO " of the toolset variable you want to use." ;
- ECHO ;
- ECHO " e.g.: set WATCOM=C:\WATCOM" ;
- ECHO " set JAM_TOOLSET=WATCOM" ;
- ECHO ;
- EXIT ;
- }
- }
-
- RM = del /f ;
- CP = copy ;
- MV ?= move ;
- DOT ?= . ;
- DOTDOT ?= .. ;
- SUFLIB ?= .lib ;
- SUFOBJ ?= .obj ;
- SUFEXE ?= .exe ;
-
- if $(TOOLSET) = WATCOM
- {
- AR ?= wlib ;
- BINDIR ?= \\os2\\apps ;
- CC ?= wcc386 ;
- CCFLAGS ?= /zq /DOS2 /I$(WATCOM)\\h ; # zq=quiet
- C++ ?= wpp386 ;
- C++FLAGS ?= $(CCFLAGS) ;
- LINK ?= wcl386 ;
- LINKFLAGS ?= /zq ; # zq=quiet
- LINKLIBS ?= ;
- NOARSCAN ?= true ;
- OPTIM ?= ;
- SLASH ?= \\ ;
- STDHDRS ?= $(WATCOM)\\h ;
- UNDEFFLAG ?= "/u _" ;
- }
- else if $(TOOLSET) = EMX
- {
- ECHO "Compiler is GCC-EMX" ;
- AR ?= ar -ru ;
- CC ?= gcc ;
- CCFLAGS ?= "" ;
- C++ ?= $(CC) ;
- C++FLAGS ?= $(CCFLAGS) ;
- LINK ?= $(CC) ;
- LINKFLAGS ?= "" ;
- LINKLIBS ?= "" ;
- OPTIM ?= ;
- SUFOBJ = .o ;
- SUFLIB = .a ;
- UNDEFFLAG ?= "-U" ;
- SLASH = / ;
-# NOARSCAN ?= true ;
- }
- else
- {
- # should never happen
- EXIT "Sorry, but the $(JAM_TOOLSET) toolset isn't supported for now" ;
- }
-}
-else if $(VMS)
-{
- C++ ?= cxx ;
- C++FLAGS ?= ;
- CC ?= cc ;
- CCFLAGS ?= ;
- CHMOD ?= set file/prot= ;
- CP ?= copy/replace ;
- CRELIB ?= true ;
- DOT ?= [] ;
- DOTDOT ?= [-] ;
- EXEMODE ?= (w:e) ;
- FILEMODE ?= (w:r) ;
- HDRS ?= ;
- LINK ?= link ;
- LINKFLAGS ?= "" ;
- LINKLIBS ?= ;
- MKDIR ?= create/dir ;
- MV ?= rename ;
- OPTIM ?= "" ;
- RM ?= delete ;
- RUNVMS ?= mcr ;
- SHELLMODE ?= (w:er) ;
- SLASH ?= . ;
- STDHDRS ?= decc$library_include ;
- SUFEXE ?= .exe ;
- SUFLIB ?= .olb ;
- SUFOBJ ?= .obj ;
-
- switch $(OS)
- {
- case OPENVMS : CCFLAGS ?= /stand=vaxc ;
- case VMS : LINKLIBS ?= sys$library:vaxcrtl.olb/lib ;
- }
-}
-else if $(MAC)
-{
- local OPT ;
-
- CW ?= "{CW}" ;
-
- MACHDRS ?=
- "$(UMACHDRS):Universal:Interfaces:CIncludes"
- "$(CW):MSL:MSL_C:MSL_Common:Include"
- "$(CW):MSL:MSL_C:MSL_MacOS:Include" ;
-
- MACLIBS ?=
- "$(CW):MacOS Support:Universal:Libraries:StubLibraries:Interfacelib"
- "$(CW):MacOS Support:Universal:Libraries:StubLibraries:Mathlib" ;
-
- MPWLIBS ?=
- "$(CW):MacOS Support:Libraries:Runtime:Runtime PPC:MSL MPWCRuntime.lib"
- "$(CW):MSL:MSL_C:MSL_MacOS:Lib:PPC:MSL C.PPC MPW.Lib" ;
-
- MPWNLLIBS ?=
- "$(CW):MacOS Support:Libraries:Runtime:Runtime PPC:MSL MPWCRuntime.lib"
- "$(CW):MSL:MSL_C:MSL_MacOS:Lib:PPC:MSL C.PPC MPW(NL).Lib" ;
-
- SIOUXHDRS ?= ;
-
- SIOUXLIBS ?=
- "$(CW):MacOS Support:Libraries:Runtime:Runtime PPC:MSL RuntimePPC.lib"
- "$(CW):MSL:MSL_C:MSL_MacOS:Lib:PPC:MSL SIOUX.PPC.Lib"
- "$(CW):MSL:MSL_C:MSL_MacOS:Lib:PPC:MSL C.PPC.Lib" ;
-
- C++ ?= mwcppc ;
- C++FLAGS ?= -w off -nomapcr ;
- CC ?= mwcppc ;
- CCFLAGS ?= -w off -nomapcr ;
- CP ?= duplicate -y ;
- DOT ?= ":" ;
- DOTDOT ?= "::" ;
- HDRS ?= $(MACHDRS) $(MPWHDRS) ;
- LINK ?= mwlinkppc ;
- LINKFLAGS ?= -mpwtool -warn ;
- LINKLIBS ?= $(MACLIBS) $(MPWLIBS) ;
- MKDIR ?= newfolder ;
- MV ?= rename -y ;
- NOARSCAN ?= true ;
- OPTIM ?= ;
- RM ?= delete -y ;
- SLASH ?= ":" ;
- STDHDRS ?= ;
- SUFLIB ?= .lib ;
- SUFOBJ ?= .o ;
-}
-else if $(OS) = BEOS && $(METROWERKS)
-{
- AR ?= mwld -xml -o ;
- BINDIR ?= /boot/apps ;
- CC ?= mwcc ;
- CCFLAGS ?= -nosyspath ;
- C++ ?= $(CC) ;
- C++FLAGS ?= -nosyspath ;
- FORTRAN ?= "" ;
- LIBDIR ?= /boot/develop/libraries ;
- LINK ?= mwld ;
- LINKFLAGS ?= "" ;
- MANDIR ?= /boot/documentation/"Shell Tools"/HTML ;
- NOARSCAN ?= true ;
- STDHDRS ?= /boot/develop/headers/posix ;
-}
-else if $(OS) = BEOS
-{
- BINDIR ?= /boot/apps ;
- CC ?= gcc ;
- C++ ?= $(CC) ;
- FORTRAN ?= "" ;
- LIBDIR ?= /boot/develop/libraries ;
- LINK ?= gcc ;
- LINKLIBS ?= -lnet ;
- NOARSCAN ?= true ;
- STDHDRS ?= /boot/develop/headers/posix ;
-}
-else if $(UNIX)
-{
- switch $(OS)
- {
- case AIX :
- LINKLIBS ?= -lbsd ;
-
- case AMIGA :
- CC ?= gcc ;
- YACC ?= "bison -y" ;
-
- case CYGWIN :
- CC ?= gcc ;
- CCFLAGS += -D__cygwin__ ;
- LEX ?= flex ;
- RANLIB ?= "" ;
- SUFEXE ?= .exe ;
- YACC ?= "bison -y" ;
-
- case DGUX :
- RANLIB ?= "" ;
- RELOCATE ?= true ;
-
- case HPUX :
- YACC = ;
- CFLAGS += -Ae ;
- CCFLAGS += -Ae ;
- RANLIB ?= "" ;
-
- case INTERIX :
- CC ?= gcc ;
- RANLIB ?= "" ;
-
- case IRIX :
- RANLIB ?= "" ;
-
- case MPEIX :
- CC ?= gcc ;
- C++ ?= gcc ;
- CCFLAGS += -D_POSIX_SOURCE ;
- HDRS += /usr/include ;
- RANLIB ?= "" ;
- NOARSCAN ?= true ;
- NOARUPDATE ?= true ;
-
- case MVS :
- RANLIB ?= "" ;
-
- case NEXT :
- AR ?= libtool -o ;
- RANLIB ?= "" ;
-
- case MACOSX :
- AR ?= libtool -o ;
- C++ ?= c++ ;
- MANDIR ?= /usr/local/share/man ;
- RANLIB ?= "" ;
-
- case NCR :
- RANLIB ?= "" ;
-
- case PTX :
- RANLIB ?= "" ;
-
- case QNX :
- AR ?= wlib ;
- CC ?= cc ;
- CCFLAGS ?= -Q ; # quiet
- C++ ?= $(CC) ;
- C++FLAGS ?= -Q ; # quiet
- LINK ?= $(CC) ;
- LINKFLAGS ?= -Q ; # quiet
- NOARSCAN ?= true ;
- RANLIB ?= "" ;
-
- case SCO :
- RANLIB ?= "" ;
- RELOCATE ?= true ;
-
- case SINIX :
- RANLIB ?= "" ;
-
- case SOLARIS :
- RANLIB ?= "" ;
- AR ?= "/usr/ccs/bin/ar ru" ;
-
- case UNICOS :
- NOARSCAN ?= true ;
- OPTIM ?= -O0 ;
-
- case UNIXWARE :
- RANLIB ?= "" ;
- RELOCATE ?= true ;
- }
-
- # UNIX defaults
-
- CCFLAGS ?= ;
- C++FLAGS ?= $(CCFLAGS) ;
- CHMOD ?= chmod ;
- CHGRP ?= chgrp ;
- CHOWN ?= chown ;
- LEX ?= lex ;
- LINKFLAGS ?= $(CCFLAGS) ;
- LINKLIBS ?= ;
- OPTIM ?= -O ;
- RANLIB ?= ranlib ;
- YACC ?= yacc ;
- YACCFILES ?= y.tab ;
- YACCFLAGS ?= -d ;
-}
-
-#
-# General defaults; a lot like UNIX
-#
-
- AR ?= ar ru ;
- AS ?= as ;
- ASFLAGS ?= ;
- AWK ?= awk ;
- BINDIR ?= /usr/local/bin ;
- C++ ?= cc ;
- C++FLAGS ?= ;
- CC ?= cc ;
- CCFLAGS ?= ;
- CP ?= cp -f ;
- CRELIB ?= ;
- DOT ?= . ;
- DOTDOT ?= .. ;
- EXEMODE ?= 711 ;
- FILEMODE ?= 644 ;
- FORTRAN ?= f77 ;
- FORTRANFLAGS ?= ;
- HDRS ?= ;
- INSTALLGRIST ?= installed ;
- JAMFILE ?= Jamfile ;
- JAMRULES ?= Jamrules ;
- LEX ?= ;
- LIBDIR ?= /usr/local/lib ;
- LINK ?= $(CC) ;
- LINKFLAGS ?= ;
- LINKLIBS ?= ;
- LN ?= ln ;
- MANDIR ?= /usr/local/man ;
- MKDIR ?= mkdir ;
- MV ?= mv -f ;
- OPTIM ?= ;
- RCP ?= rcp ;
- RM ?= rm -f ;
- RSH ?= rsh ;
- SED ?= sed ;
- SHELLHEADER ?= "#!/bin/sh" ;
- SHELLMODE ?= 755 ;
- SLASH ?= / ;
- STDHDRS ?= /usr/include ;
- SUFEXE ?= "" ;
- SUFLIB ?= .a ;
- SUFOBJ ?= .o ;
- UNDEFFLAG ?= "-u _" ;
- YACC ?= ;
- YACCFILES ?= ;
- YACCFLAGS ?= ;
-
- HDRPATTERN =
- "^[ ]*#[ ]*include[ ]*[<\"]([^\">]*)[\">].*$" ;
-
- OSFULL = $(OS)$(OSVER)$(OSPLAT) $(OS)$(OSPLAT) $(OS)$(OSVER) $(OS) ;
-
-
-#
-# Base dependencies - first for "bootstrap" kinds of rules
-#
-
-DEPENDS all : shell files lib exe obj ;
-DEPENDS all shell files lib exe obj : first ;
-NOTFILE all first shell files lib exe obj dirs clean uninstall ;
-ALWAYS clean uninstall ;
-
-#
-# Rules
-#
-
-rule As
-{
- DEPENDS $(<) : $(>) ;
- ASFLAGS on $(<) += $(ASFLAGS) $(SUBDIRASFLAGS) ;
-}
-
-rule Bulk
-{
- local i ;
-
- for i in $(>)
- {
- File $(i:D=$(<)) : $(i) ;
- }
-}
-
-rule Cc
-{
- local _h ;
-
- DEPENDS $(<) : $(>) ;
-
- # Just to clarify here: this sets the per-target CCFLAGS to
- # be the current value of (global) CCFLAGS and SUBDIRCCFLAGS.
-
- CCFLAGS on $(<) += $(CCFLAGS) $(SUBDIRCCFLAGS) ;
-
- # If the compiler's -o flag doesn't work, relocate the .o
-
- if $(RELOCATE)
- {
- CcMv $(<) : $(>) ;
- }
-
- _h = $(SEARCH_SOURCE) $(HDRS) $(SUBDIRHDRS) ;
-
- if $(VMS) && $(_h)
- {
- SLASHINC on $(<) = "/inc=(" $(_h[1]) ,$(_h[2-]) ")" ;
- }
- else if $(MAC) && $(_h)
- {
- local _i _j ;
- _j = $(_h[1]) ;
- for _i in $(_h[2-])
- {
- _j = $(_j),$(_i) ;
- }
- MACINC on $(<) = \"$(_j)\" ;
- }
-}
-
-rule C++
-{
- local _h ;
-
- DEPENDS $(<) : $(>) ;
- C++FLAGS on $(<) += $(C++FLAGS) $(SUBDIRC++FLAGS) ;
-
- if $(RELOCATE)
- {
- CcMv $(<) : $(>) ;
- }
-
- _h = $(SEARCH_SOURCE) $(HDRS) $(SUBDIRHDRS) ;
-
- if $(VMS) && $(_h)
- {
- SLASHINC on $(<) = "/inc=(" $(_h[1]) ,$(_h[2-]) ")" ;
- }
- else if $(MAC) && $(_h)
- {
- local _i _j ;
- _j = $(_h[1]) ;
- for _i in $(_h[2-])
- {
- _j = $(_j),$(_i) ;
- }
- MACINC on $(<) = \"$(_j)\" ;
- }
-}
-
-rule Chmod
-{
- if $(CHMOD) { Chmod1 $(<) ; }
-}
-
-rule File
-{
- DEPENDS files : $(<) ;
- DEPENDS $(<) : $(>) ;
- SEARCH on $(>) = $(SEARCH_SOURCE) ;
- MODE on $(<) = $(FILEMODE) ;
- Chmod $(<) ;
-}
-
-rule Fortran
-{
- DEPENDS $(<) : $(>) ;
-}
-
-rule GenFile
-{
- local _t = [ FGristSourceFiles $(<) ] ;
- local _s = [ FAppendSuffix $(>[1]) : $(SUFEXE) ] ;
- Depends $(_t) : $(_s) $(>[2-]) ;
- GenFile1 $(_t) : $(_s) $(>[2-]) ;
- Clean clean : $(_t) ;
-}
-
-rule GenFile1
-{
- MakeLocate $(<) : $(LOCATE_SOURCE) ;
- SEARCH on $(>) = $(SEARCH_SOURCE) ;
-}
-
-rule HardLink
-{
- DEPENDS files : $(<) ;
- DEPENDS $(<) : $(>) ;
- SEARCH on $(>) = $(SEARCH_SOURCE) ;
-}
-
-rule HdrMacroFile
-{
- # HdrMacroFile file ;
- #
- # this rule is used to indicate that a given file contains definitions
- # for filename macros (e.g. "#define MYFILE_H <myfile.h>") that can
- # later be used in #include statements in the rest of the source
- #
- # theses files must be parsed before any make is tried..
- #
- HDRMACRO $(<) ;
-}
-
-rule HdrRule
-{
- # HdrRule source : headers ;
-
- # N.B. This rule is called during binding, potentially after
- # the fate of many targets has been determined, and must be
- # used with caution: don't add dependencies to unrelated
- # targets, and don't set variables on $(<).
-
- # Tell Jam that anything depending on $(<) also depends on $(>),
- # set SEARCH so Jam can find the headers, but then say we don't
- # care if we can't actually find the headers (they may have been
- # within ifdefs),
-
- local s ;
-
- if $(HDRGRIST)
- {
- s = $(>:G=$(HDRGRIST)) ;
- } else {
- s = $(>) ;
- }
-
- INCLUDES $(<) : $(s) ;
- SEARCH on $(s) = $(HDRSEARCH) ;
- NOCARE $(s) ;
-
- # Propagate on $(<) to $(>)
-
- HDRSEARCH on $(s) = $(HDRSEARCH) ;
- HDRSCAN on $(s) = $(HDRSCAN) ;
- HDRRULE on $(s) = $(HDRRULE) ;
- HDRGRIST on $(s) = $(HDRGRIST) ;
-}
-
-rule InstallInto
-{
- # InstallInto dir : sources ;
-
- local i t ;
-
- t = $(>:G=$(INSTALLGRIST)) ;
-
- # Arrange for jam install
- # Arrange for jam uninstall
- # sources are in SEARCH_SOURCE
- # targets are in dir
-
- Depends install : $(t) ;
- Clean uninstall : $(t) ;
- SEARCH on $(>) = $(SEARCH_SOURCE) ;
- MakeLocate $(t) : $(<) ;
-
- # For each source, make gristed target name
- # and Install, Chmod, Chown, and Chgrp
-
- for i in $(>)
- {
- local tt = $(i:G=$(INSTALLGRIST)) ;
-
- Depends $(tt) : $(i) ;
- Install $(tt) : $(i) ;
- Chmod $(tt) ;
-
- if $(OWNER) && $(CHOWN)
- {
- Chown $(tt) ;
- OWNER on $(tt) = $(OWNER) ;
- }
-
- if $(GROUP) && $(CHGRP)
- {
- Chgrp $(tt) ;
- GROUP on $(tt) = $(GROUP) ;
- }
- }
-}
-
-rule InstallBin
-{
- local _t = [ FAppendSuffix $(>) : $(SUFEXE) ] ;
-
- InstallInto $(<) : $(_t) ;
- MODE on $(_t:G=installed) = $(EXEMODE) ;
-}
-
-rule InstallFile
-{
- InstallInto $(<) : $(>) ;
- MODE on $(>:G=installed) = $(FILEMODE) ;
-}
-
-rule InstallLib
-{
- InstallInto $(<) : $(>) ;
- MODE on $(>:G=installed) = $(FILEMODE) ;
-}
-
-rule InstallMan
-{
- # Really this just strips the . from the suffix
-
- local i s d ;
-
- for i in $(>)
- {
- switch $(i:S)
- {
- case .1 : s = 1 ; case .2 : s = 2 ; case .3 : s = 3 ;
- case .4 : s = 4 ; case .5 : s = 5 ; case .6 : s = 6 ;
- case .7 : s = 7 ; case .8 : s = 8 ; case .l : s = l ;
- case .n : s = n ; case .man : s = 1 ;
- }
-
- d = man$(s) ;
-
- InstallInto $(d:R=$(<)) : $(i) ;
- }
-
- MODE on $(>:G=installed) = $(FILEMODE) ;
-}
-
-rule InstallShell
-{
- InstallInto $(<) : $(>) ;
- MODE on $(>:G=installed) = $(SHELLMODE) ;
-}
-
-rule Lex
-{
- LexMv $(<) : $(>) ;
- DEPENDS $(<) : $(>) ;
- MakeLocate $(<) : $(LOCATE_SOURCE) ;
- Clean clean : $(<) ;
-}
-
-rule Library
-{
- LibraryFromObjects $(<) : $(>:S=$(SUFOBJ)) ;
- Objects $(>) ;
-}
-
-rule LibraryFromObjects
-{
- local _i _l _s ;
-
- # Add grist to file names
-
- _s = [ FGristFiles $(>) ] ;
- _l = $(<:S=$(SUFLIB)) ;
-
- # library depends on its member objects
-
- if $(KEEPOBJS)
- {
- DEPENDS obj : $(_s) ;
- }
- else
- {
- DEPENDS lib : $(_l) ;
- }
-
- # Set LOCATE for the library and its contents. The bound
- # value shows up as $(NEEDLIBS) on the Link actions.
- # For compatibility, we only do this if the library doesn't
- # already have a path.
-
- if ! $(_l:D)
- {
- MakeLocate $(_l) $(_l)($(_s:BS)) : $(LOCATE_TARGET) ;
- }
-
- if $(NOARSCAN)
- {
- # If we can't scan the library to timestamp its contents,
- # we have to just make the library depend directly on the
- # on-disk object files.
-
- DEPENDS $(_l) : $(_s) ;
- }
- else
- {
- # If we can scan the library, we make the library depend
- # on its members and each member depend on the on-disk
- # object file.
-
- DEPENDS $(_l) : $(_l)($(_s:BS)) ;
-
- for _i in $(_s)
- {
- DEPENDS $(_l)($(_i:BS)) : $(_i) ;
- }
- }
-
- Clean clean : $(_l) ;
-
- if $(CRELIB) { CreLib $(_l) : $(_s[1]) ; }
-
- Archive $(_l) : $(_s) ;
-
- if $(RANLIB) { Ranlib $(_l) ; }
-
- # If we can't scan the library, we have to leave the .o's around.
-
- if ! ( $(NOARSCAN) || $(KEEPOBJS) ) { RmTemps $(_l) : $(_s) ; }
-}
-
-rule Link
-{
- MODE on $(<) = $(EXEMODE) ;
- Chmod $(<) ;
-}
-
-rule LinkLibraries
-{
- # make library dependencies of target
- # set NEEDLIBS variable used by 'actions Main'
-
- local _t = [ FAppendSuffix $(<) : $(SUFEXE) ] ;
-
- DEPENDS $(_t) : $(>:S=$(SUFLIB)) ;
- NEEDLIBS on $(_t) += $(>:S=$(SUFLIB)) ;
-}
-
-rule Main
-{
- MainFromObjects $(<) : $(>:S=$(SUFOBJ)) ;
- Objects $(>) ;
-}
-
-rule MainFromObjects
-{
- local _s _t ;
-
- # Add grist to file names
- # Add suffix to exe
-
- _s = [ FGristFiles $(>) ] ;
- _t = [ FAppendSuffix $(<) : $(SUFEXE) ] ;
-
- if $(_t) != $(<)
- {
- DEPENDS $(<) : $(_t) ;
- NOTFILE $(<) ;
- }
-
- # make compiled sources a dependency of target
-
- DEPENDS exe : $(_t) ;
- DEPENDS $(_t) : $(_s) ;
- MakeLocate $(_t) : $(LOCATE_TARGET) ;
-
- Clean clean : $(_t) ;
-
- Link $(_t) : $(_s) ;
-}
-
-rule MakeLocate
-{
- if $(>)
- {
- LOCATE on $(<) = $(>) ;
- Depends $(<) : $(>[1]) ;
- MkDir $(>[1]) ;
- }
-}
-
-rule MkDir
-{
- # If dir exists, don't update it
- # Do this even for $(DOT).
-
- NOUPDATE $(<) ;
-
- if $(<) != $(DOT) && ! $($(<)-mkdir)
- {
- local s ;
-
- # Cheesy gate to prevent multiple invocations on same dir
- # MkDir1 has the actions
- # Arrange for jam dirs
-
- $(<)-mkdir = true ;
- MkDir1 $(<) ;
- Depends dirs : $(<) ;
-
- # Recursively make parent directories.
- # $(<:P) = $(<)'s parent, & we recurse until root
-
- s = $(<:P) ;
-
- if $(NT)
- {
- switch $(s)
- {
- case *: : s = ;
- case *:\\ : s = ;
- }
- }
-
- if $(s) && $(s) != $(<)
- {
- Depends $(<) : $(s) ;
- MkDir $(s) ;
- }
- else if $(s)
- {
- NOTFILE $(s) ;
- }
-
- }
-}
-
-rule Object
-{
- local h ;
-
- # locate object and search for source, if wanted
-
- Clean clean : $(<) ;
-
- MakeLocate $(<) : $(LOCATE_TARGET) ;
- SEARCH on $(>) = $(SEARCH_SOURCE) ;
-
- # Save HDRS for -I$(HDRS) on compile.
- # We shouldn't need -I$(SEARCH_SOURCE) as cc can find headers
- # in the .c file's directory, but generated .c files (from
- # yacc, lex, etc) are located in $(LOCATE_TARGET), possibly
- # different from $(SEARCH_SOURCE).
-
- HDRS on $(<) = $(SEARCH_SOURCE) $(HDRS) $(SUBDIRHDRS) ;
-
- # handle #includes for source: Jam scans for headers with
- # the regexp pattern $(HDRSCAN) and then invokes $(HDRRULE)
- # with the scanned file as the target and the found headers
- # as the sources. HDRSEARCH is the value of SEARCH used for
- # the found header files. Finally, if jam must deal with
- # header files of the same name in different directories,
- # they can be distinguished with HDRGRIST.
-
- # $(h) is where cc first looks for #include "foo.h" files.
- # If the source file is in a distant directory, look there.
- # Else, look in "" (the current directory).
-
- if $(SEARCH_SOURCE)
- {
- h = $(SEARCH_SOURCE) ;
- }
- else
- {
- h = "" ;
- }
-
- HDRRULE on $(>) = HdrRule ;
- HDRSCAN on $(>) = $(HDRPATTERN) ;
- HDRSEARCH on $(>) = $(HDRS) $(SUBDIRHDRS) $(h) $(STDHDRS) ;
- HDRGRIST on $(>) = $(HDRGRIST) ;
-
- # if source is not .c, generate .c with specific rule
-
- switch $(>:S)
- {
- case .asm : As $(<) : $(>) ;
- case .c : Cc $(<) : $(>) ;
- case .C : C++ $(<) : $(>) ;
- case .cc : C++ $(<) : $(>) ;
- case .cpp : C++ $(<) : $(>) ;
- case .f : Fortran $(<) : $(>) ;
- case .l : Cc $(<) : $(<:S=.c) ;
- Lex $(<:S=.c) : $(>) ;
- case .s : As $(<) : $(>) ;
- case .y : Cc $(<) : $(<:S=.c) ;
- Yacc $(<:S=.c) : $(>) ;
- case * : UserObject $(<) : $(>) ;
- }
-}
-
-
-rule ObjectCcFlags
-{
- CCFLAGS on [ FGristFiles $(<:S=$(SUFOBJ)) ] += $(>) ;
-}
-
-rule ObjectC++Flags
-{
- C++FLAGS on [ FGristFiles $(<:S=$(SUFOBJ)) ] += $(>) ;
-}
-
-rule ObjectHdrs
-{
- HDRS on [ FGristFiles $(<:S=$(SUFOBJ)) ] += $(>) ;
-}
-
-rule Objects
-{
- local _i ;
-
- for _i in [ FGristFiles $(<) ]
- {
- Object $(_i:S=$(SUFOBJ)) : $(_i) ;
- DEPENDS obj : $(_i:S=$(SUFOBJ)) ;
- }
-}
-
-rule RmTemps
-{
- TEMPORARY $(>) ;
-}
-
-rule Setuid
-{
- MODE on [ FAppendSuffix $(<) : $(SUFEXE) ] = 4711 ;
-}
-
-rule Shell
-{
- DEPENDS shell : $(<) ;
- DEPENDS $(<) : $(>) ;
- SEARCH on $(>) = $(SEARCH_SOURCE) ;
- MODE on $(<) = $(SHELLMODE) ;
- Clean clean : $(<) ;
- Chmod $(<) ;
-}
-
-rule SubDir
-{
- local _r _s ;
-
- #
- # SubDir TOP d1 [ ... ]
- #
- # This introduces a Jamfile that is part of a project tree
- # rooted at $(TOP). It (only once) includes the project-specific
- # rules file $(TOP)/Jamrules and then sets search & locate stuff.
- #
- # If the variable $(TOPRULES) is set (where TOP is the first arg
- # to SubDir), that file is included instead of $(TOP)/Jamrules.
- #
- # d1 ... are the directory elements that lead to this directory
- # from $(TOP). We construct the system dependent path from these
- # directory elements in order to set search & locate stuff.
- #
-
- if ! $($(<[1]))
- {
- if ! $(<[1])
- {
- EXIT SubDir syntax error ;
- }
-
- $(<[1]) = [ FSubDir $(<[2-]) ] ;
- }
-
- #
- # If $(TOP)/Jamrules hasn't been included, do so.
- #
-
- if ! $($(<[1])-included)
- {
- # Gated entry.
-
- $(<[1])-included = TRUE ;
-
- # File is $(TOPRULES) or $(TOP)/Jamrules.
-
- _r = $($(<[1])RULES) ;
-
- if ! $(_r)
- {
- _r = $(JAMRULES:R=$($(<[1]))) ;
- }
-
- # Include it.
-
- include $(_r) ;
- }
-
- # Get path to current directory from root using SubDir.
- # Save dir tokens for other potential uses.
-
- _s = [ FDirName $(<[2-]) ] ;
- SUBDIR = $(_s:R=$($(<[1]))) ;
- SUBDIR_TOKENS = $(<[2-]) ;
-
- # Now set up SEARCH_SOURCE, LOCATE_TARGET, SOURCE_GRIST
- # These can be reset if needed. For example, if the source
- # directory should not hold object files, LOCATE_TARGET can
- # subsequently be redefined.
-
- SEARCH_SOURCE = $(SUBDIR) ;
- LOCATE_SOURCE = $(ALL_LOCATE_TARGET) $(SUBDIR) ;
- LOCATE_TARGET = $(ALL_LOCATE_TARGET) $(SUBDIR) ;
- SOURCE_GRIST = [ FGrist $(<[2-]) ] ;
-
- # Reset per-directory ccflags, hdrs
-
- SUBDIRCCFLAGS = ;
- SUBDIRC++FLAGS = ;
- SUBDIRHDRS = ;
-}
-
-rule SubDirCcFlags
-{
- SUBDIRCCFLAGS += $(<) ;
-}
-
-rule SubDirC++Flags
-{
- SUBDIRC++FLAGS += $(<) ;
-}
-
-rule SubDirHdrs
-{
- SUBDIRHDRS += $(<) ;
-}
-
-rule SubInclude
-{
- local _s ;
-
- # That's
- # SubInclude TOP d1 [ d2 [ d3 [ d4 ] ] ]
- #
- # to include a subdirectory's Jamfile.
-
- if ! $($(<[1]))
- {
- EXIT Top level of source tree has not been set with $(<[1]) ;
- }
-
- _s = [ FDirName $(<[2-]) ] ;
-
- include $(JAMFILE:D=$(_s):R=$($(<[1]))) ;
-}
-
-rule Undefines
-{
- UNDEFS on [ FAppendSuffix $(<) : $(SUFEXE) ] += $(UNDEFFLAG)$(>) ;
-}
-
-rule UserObject
-{
- EXIT "Unknown suffix on" $(>) "- see UserObject rule in Jamfile(5)." ;
-}
-
-rule Yacc
-{
- local _h ;
-
- _h = $(<:BS=.h) ;
-
- # Some places don't have a yacc.
-
- MakeLocate $(<) $(_h) : $(LOCATE_SOURCE) ;
-
- if $(YACC)
- {
- DEPENDS $(<) $(_h) : $(>) ;
- Yacc1 $(<) $(_h) : $(>) ;
- YaccMv $(<) $(_h) : $(>) ;
- Clean clean : $(<) $(_h) ;
- }
-
- # Make sure someone includes $(_h) else it will be a deadly independent
- # target.
- INCLUDES $(<) : $(_h) ;
-}
-
-#
-# Utility rules; no side effects on these.
-#
-
-rule FGrist
-{
- # Turn individual elements in $(<) into grist.
-
- local _g _i ;
-
- _g = $(<[1]) ;
-
- for _i in $(<[2-])
- {
- _g = $(_g)!$(_i) ;
- }
-
- return $(_g) ;
-}
-
-rule FGristFiles
-{
- if ! $(SOURCE_GRIST)
- {
- return $(<) ;
- }
- else
- {
- return $(<:G=$(SOURCE_GRIST)) ;
- }
-}
-
-rule FGristSourceFiles
-{
- # Produce source file name name with grist in it,
- # if SOURCE_GRIST is set.
-
- # Leave header files alone, because they have a global
- # visibility.
-
- if ! $(SOURCE_GRIST)
- {
- return $(<) ;
- }
- else
- {
- local _i _o ;
-
- for _i in $(<)
- {
- switch $(_i)
- {
- case *.h : _o += $(_i) ;
- case * : _o += $(_i:G=$(SOURCE_GRIST)) ;
- }
- }
-
- return $(_o) ;
- }
-}
-
-rule FConcat
-{
- # Puts the variables together, removing spaces.
-
- local _t _r ;
-
- $(_r) = $(<[1]) ;
-
- for _t in $(<[2-])
- {
- $(_r) = $(_r)$(_t) ;
- }
-
- return $(_r) ;
-}
-
-rule FSubDir
-{
- local _i _d ;
-
- # If $(>) is the path to the current directory, compute the
- # path (using ../../ etc) back to that root directory.
- # Sets result in $(<)
-
- if ! $(<[1])
- {
- _d = $(DOT) ;
- }
- else
- {
- _d = $(DOTDOT) ;
-
- for _i in $(<[2-])
- {
- _d = $(_d:R=$(DOTDOT)) ;
- }
- }
-
- return $(_d) ;
-}
-
-rule FDirName
-{
- local _s _i ;
-
- # Turn individual elements in $(<) into a usable path.
-
- if ! $(<)
- {
- _s = $(DOT) ;
- }
- else if $(VMS)
- {
- # This handles the following cases:
- # a -> [.a]
- # a b c -> [.a.b.c]
- # x: -> x:
- # x: a -> x:[a]
- # x:[a] b -> x:[a.b]
-
- switch $(<[1])
- {
- case *:* : _s = $(<[1]) ;
- case \\[*\\] : _s = $(<[1]) ;
- case * : _s = [.$(<[1])] ;
- }
-
- for _i in [.$(<[2-])]
- {
- _s = $(_i:R=$(_s)) ;
- }
- }
- else if $(MAC)
- {
- _s = $(DOT) ;
-
- for _i in $(<)
- {
- _s = $(_i:R=$(_s)) ;
- }
- }
- else
- {
- _s = $(<[1]) ;
-
- for _i in $(<[2-])
- {
- _s = $(_i:R=$(_s)) ;
- }
- }
-
- return $(_s) ;
-}
-
-
-rule _makeCommon
-{
- # strip common initial elements
-
- if $($(<)[1]) && $($(<)[1]) = $($(>)[1])
- {
- $(<) = $($(<)[2-]) ;
- $(>) = $($(>)[2-]) ;
- _makeCommon $(<) : $(>) ;
- }
-}
-
-
-rule FRelPath
-{
- local _l _r ;
-
- # first strip off common parts
-
- _l = $(<) ;
- _r = $(>) ;
-
- _makeCommon _l : _r ;
-
- # now make path to root and path down
-
- _l = [ FSubDir $(_l) ] ;
- _r = [ FDirName $(_r) ] ;
-
- # Concatenate and save
-
- # XXX This should be better
-
- if $(_r) = $(DOT) {
- return $(_l) ;
- } else {
- return $(_r:R=$(_l)) ;
- }
-}
-
-rule FAppendSuffix
-{
- # E.g., "FAppendSuffix yacc lex foo.bat : $(SUFEXE) ;"
- # returns (yacc,lex,foo.bat) on Unix and
- # (yacc.exe,lex.exe,foo.bat) on NT.
-
- if $(>)
- {
- local _i _o ;
-
- for _i in $(<)
- {
- if $(_i:S)
- {
- _o += $(_i) ;
- }
- else
- {
- _o += $(_i:S=$(>)) ;
- }
- }
- return $(_o) ;
- }
- else
- {
- return $(<) ;
- }
-}
-
-rule unmakeDir
-{
- if $(>[1]:D) && $(>[1]:D) != $(>[1]) && $(>[1]:D) != \\\\
- {
- unmakeDir $(<) : $(>[1]:D) $(>[1]:BS) $(>[2-]) ;
- }
- else
- {
- $(<) = $(>) ;
- }
-}
-
-
-rule FConvertToSlashes
-{
- local _d, _s, _i ;
-
- unmakeDir _d : $(<) ;
-
- _s = $(_d[1]) ;
- for _i in $(_d[2-])
- {
- _s = $(_s)/$(_i) ;
- }
- return $(_s) ;
-}
-
-
-#
-# Actions
-#
-
-#
-# First the defaults
-#
-
-actions updated together piecemeal Archive
-{
- $(AR) $(<) $(>)
-}
-
-actions As
-{
- $(AS) $(ASFLAGS) -I$(HDRS) -o $(<) $(>)
-}
-
-actions C++
-{
- $(C++) -c $(C++FLAGS) $(OPTIM) -I$(HDRS) -o $(<) $(>)
-}
-
-actions Cc
-{
- $(CC) -c $(CCFLAGS) $(OPTIM) -I$(HDRS) -o $(<) $(>)
-}
-
-actions Chgrp
-{
- $(CHGRP) $(GROUP) $(<)
-}
-
-actions Chmod1
-{
- $(CHMOD) $(MODE) $(<)
-}
-
-actions Chown
-{
- $(CHOWN) $(OWNER) $(<)
-}
-
-actions piecemeal together existing Clean
-{
- $(RM) $(>)
-}
-
-actions File
-{
- $(CP) $(>) $(<)
-}
-
-actions GenFile1
-{
- $(>[1]) $(<) $(>[2-])
-}
-
-actions Fortran
-{
- $(FORTRAN) $(FORTRANFLAGS) -o $(<) $(>)
-}
-
-actions HardLink
-{
- $(RM) $(<) && $(LN) $(>) $(<)
-}
-
-actions Install
-{
- $(CP) $(>) $(<)
-}
-
-actions Lex
-{
- $(LEX) $(>)
-}
-
-actions LexMv
-{
- $(MV) lex.yy.c $(<)
-}
-
-actions Link bind NEEDLIBS
-{
- $(LINK) $(LINKFLAGS) -o $(<) $(UNDEFS) $(>) $(NEEDLIBS) $(LINKLIBS)
-}
-
-actions MkDir1
-{
- $(MKDIR) $(<)
-}
-
-actions together Ranlib
-{
- $(RANLIB) $(<)
-}
-
-actions quietly updated piecemeal together RmTemps
-{
- $(RM) $(>)
-}
-
-actions Shell
-{
- $(AWK) '
- NR == 1 { print "$(SHELLHEADER)" }
- NR == 1 && /^[#:]/ { next }
- /^##/ { next }
- { print }
- ' < $(>) > $(<)
-}
-
-actions Yacc1
-{
- $(YACC) $(YACCFLAGS) $(>)
-}
-
-actions YaccMv
-{
- $(MV) $(YACCFILES).c $(<[1])
- $(MV) $(YACCFILES).h $(<[2])
-}
-
-#
-# RELOCATE - for compilers with broken -o flags
-#
-
-if $(RELOCATE)
-{
- actions C++
- {
- $(C++) -c $(C++FLAGS) $(OPTIM) -I$(HDRS) $(>)
- }
-
- actions Cc
- {
- $(CC) -c $(CCFLAGS) $(OPTIM) -I$(HDRS) $(>)
- }
-
- actions ignore CcMv
- {
- [ $(<) != $(>:BS=$(SUFOBJ)) ] && $(MV) $(>:BS=$(SUFOBJ)) $(<)
- }
-}
-
-#
-# NOARUPDATE - can't update an archive
-#
-
-if $(NOARUPDATE)
-{
- actions Archive
- {
- $(AR) $(<) $(>)
- }
-}
-
-#
-# NT specific actions
-#
-
-if $(NT)
-{
- if $(TOOLSET) = VISUALC || $(TOOLSET) = VC7 || $(TOOLSET) = INTELC
- {
- actions updated together piecemeal Archive
- {
- if exist $(<) set _$(<:B)_=$(<)
- $(AR) /out:$(<) %_$(<:B)_% $(>)
- }
-
- actions As
- {
- $(AS) /Ml /p /v /w2 $(>) $(<) ,nul,nul;
- }
-
- actions Cc
- {
- $(CC) /c $(CCFLAGS) $(OPTIM) /Fo$(<) /I$(HDRS) /I$(STDHDRS) $(>)
- }
-
- actions C++
- {
- $(C++) /c $(C++FLAGS) $(OPTIM) /Fo$(<) /I$(HDRS) /I$(STDHDRS) /Tp$(>)
- }
-
- actions Link bind NEEDLIBS
- {
- $(LINK) $(LINKFLAGS) /out:$(<) $(UNDEFS) $(>) $(NEEDLIBS) $(LINKLIBS)
- }
- }
- else if $(TOOLSET) = VISUALC16
- {
- actions updated together piecemeal Archive
- {
- $(AR) $(<) -+$(>)
- }
-
- actions Cc
- {
- $(CC) /c $(CCFLAGS) $(OPTIM) /Fo$(<) /I$(HDRS) $(>)
- }
-
- actions C++
- {
- $(C++) /c $(C++FLAGS) $(OPTIM) /Fo$(<) /I$(HDRS) /Tp$(>)
- }
-
- actions Link bind NEEDLIBS
- {
- $(LINK) $(LINKFLAGS) /out:$(<) $(UNDEFS) $(>) $(NEEDLIBS) $(LINKLIBS)
- }
- }
- else if $(TOOLSET) = BORLANDC
- {
- actions updated together piecemeal Archive
- {
- $(AR) $(<) -+$(>)
- }
-
- actions Link bind NEEDLIBS
- {
- $(LINK) -e$(<) $(LINKFLAGS) $(UNDEFS) -L$(LINKLIBS) $(NEEDLIBS) $(>)
- }
-
- actions C++
- {
- $(C++) -c $(C++FLAGS) $(OPTIM) -I$(HDRS) -o$(<) $(>)
- }
-
- actions Cc
- {
- $(CC) -c $(CCFLAGS) $(OPTIM) -I$(HDRS) -o$(<) $(>)
- }
-
- }
- else if $(TOOLSET) = MINGW
- {
- actions together piecemeal Archive
- {
- $(AR) $(<) $(>:T)
- }
-
- actions Cc
- {
- $(CC) -c $(CCFLAGS) $(OPTIM) -I$(HDRS) -o$(<) $(>)
- }
-
- actions C++
- {
- $(C++) -c $(C++FLAGS) $(OPTIM) -I$(HDRS) -o$(<) $(>)
- }
- }
- else if $(TOOLSET) = WATCOM
- {
- actions together piecemeal Archive
- {
- $(AR) $(<) +-$(>)
- }
-
- actions Cc
- {
- $(CC) $(CCFLAGS) $(OPTIM) /Fo=$(<) /I$(HDRS) $(>)
- }
-
- actions C++
- {
- $(C++) $(C++FLAGS) $(OPTIM) /Fo=$(<) /I$(HDRS) $(>)
- }
-
- actions Link bind NEEDLIBS
- {
- $(LINK) $(LINKFLAGS) /Fe=$(<) $(UNDEFS) $(>) $(NEEDLIBS) $(LINKLIBS)
- }
-
- actions Shell
- {
- $(CP) $(>) $(<)
- }
- }
- else if $(TOOLSET) = LCC
- {
- actions together piecemeal Archive
- {
- $(AR) /out:$(<) $(>)
- }
-
- actions Cc
- {
- $(CC) $(CCFLAGS) $(OPTIM) -Fo$(<) -I$(HDRS) $(>)
- }
-
- actions Link bind NEEDLIBS
- {
- $(LINK) $(LINKFLAGS) -o $(<) $(UNDEFS) $(>) $(NEEDLIBS) $(LINKLIBS)
- }
-
- actions Shell
- {
- $(CP) $(>) $(<)
- }
- }
-}
-
-#
-# OS2 specific actions
-#
-
-else if $(OS2)
-{
- if $(TOOLSET) = WATCOM
- {
- actions together piecemeal Archive
- {
- $(AR) $(<) +-$(>)
- }
-
- actions Cc
- {
- $(CC) $(CCFLAGS) $(OPTIM) /Fo=$(<) /I$(HDRS) $(>)
- }
-
- actions C++
- {
- $(C++) $(C++FLAGS) $(OPTIM) /Fo=$(<) /I$(HDRS) $(>)
- }
-
- actions Link bind NEEDLIBS
- {
- $(LINK) $(LINKFLAGS) /Fe=$(<) $(UNDEFS) $(>) $(NEEDLIBS) $(LINKLIBS)
- }
-
- actions Shell
- {
- $(CP) $(>) $(<)
- }
- }
- else if $(TOOLSET) = EMX
- {
- actions together piecemeal Archive
- {
- $(AR) $(<) $(>:T)
- }
-
- actions Cc
- {
- $(CC) -c $(CCFLAGS) $(OPTIM) -I$(HDRS) -o$(<) $(>)
- }
-
- actions C++
- {
- $(C++) -c $(C++FLAGS) $(OPTIM) -I$(HDRS) -o$(<) $(>)
- }
- }
-}
-
-#
-# VMS specific actions
-#
-
-else if $(VMS)
-{
- actions updated together piecemeal Archive
- {
- lib/replace $(<) $(>[1]) ,$(>[2-])
- }
-
- actions Cc
- {
- $(CC)/obj=$(<) $(CCFLAGS) $(OPTIM) $(SLASHINC) $(>)
- }
-
- actions C++
- {
- $(C++)/obj=$(<) $(C++FLAGS) $(OPTIM) $(SLASHINC) $(>)
- }
-
- actions piecemeal together existing Clean
- {
- $(RM) $(>[1]);* ,$(>[2-]);*
- }
-
- actions together quietly CreLib
- {
- if f$search("$(<)") .eqs. "" then lib/create $(<)
- }
-
- actions GenFile1
- {
- mcr $(>[1]) $(<) $(>[2-])
- }
-
- actions Link bind NEEDLIBS
- {
- $(LINK)/exe=$(<) $(LINKFLAGS) $(>[1]) ,$(>[2-]) ,$(NEEDLIBS)/lib ,$(LINKLIBS)
- }
-
- actions quietly updated piecemeal together RmTemps
- {
- $(RM) $(>[1]);* ,$(>[2-]);*
- }
-
- actions Shell
- {
- $(CP) $(>) $(<)
- }
-}
-
-#
-# Mac specifc actions
-#
-
-else if $(MAC)
-{
- actions together Archive
- {
- $(LINK) -library -o $(<) $(>)
- }
-
- actions Cc
- {
- set -e MWCincludes $(MACINC)
- $(CC) -o $(<) $(CCFLAGS) $(OPTIM) $(>)
- }
-
- actions C++
- {
- set -e MWCincludes $(MACINC)
- $(CC) -o $(<) $(C++FLAGS) $(OPTIM) $(>)
- }
-
- actions Link bind NEEDLIBS
- {
- $(LINK) -o $(<) $(LINKFLAGS) $(>) $(NEEDLIBS) "$(LINKLIBS)"
- }
-}
-
-#
-# Backwards compatibility with jam 1, where rules were uppercased.
-#
-
-rule BULK { Bulk $(<) : $(>) ; }
-rule FILE { File $(<) : $(>) ; }
-rule HDRRULE { HdrRule $(<) : $(>) ; }
-rule INSTALL { Install $(<) : $(>) ; }
-rule LIBRARY { Library $(<) : $(>) ; }
-rule LIBS { LinkLibraries $(<) : $(>) ; }
-rule LINK { Link $(<) : $(>) ; }
-rule MAIN { Main $(<) : $(>) ; }
-rule SETUID { Setuid $(<) ; }
-rule SHELL { Shell $(<) : $(>) ; }
-rule UNDEFINES { Undefines $(<) : $(>) ; }
-
-# Old INSTALL* didn't take dest directory.
-
-rule INSTALLBIN { InstallBin $(BINDIR) : $(<) ; }
-rule INSTALLLIB { InstallLib $(LIBDIR) : $(<) ; }
-rule INSTALLMAN { InstallMan $(MANDIR) : $(<) ; }
-
-# Compatibility with jam 2.2.
-
-rule addDirName { $(<) += [ FDirName $(>) ] ; }
-rule makeDirName { $(<) = [ FDirName $(>) ] ; }
-rule makeGristedName { $(<) = [ FGristSourceFiles $(>) ] ; }
-rule makeRelPath { $(<[1]) = [ FRelPath $(<[2-]) : $(>) ] ; }
-rule makeSuffixed { $(<[1]) = [ FAppendSuffix $(>) : $(<[2]) ] ; }
-
-#
-# Now include the user's Jamfile.
-#
-
-{
- if $(JAMFILE) { include $(JAMFILE) ; }
-}
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/boost-jam.spec
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/boost-jam.spec b/ext/kenlm/jam-files/engine/boost-jam.spec
deleted file mode 100644
index bc572fc..0000000
--- a/ext/kenlm/jam-files/engine/boost-jam.spec
+++ /dev/null
@@ -1,64 +0,0 @@
-Name: boost-jam
-Version: 3.1.19
-Summary: Build tool
-Release: 1
-Source: %{name}-%{version}.tgz
-
-License: Boost Software License, Version 1.0
-Group: Development/Tools
-URL: http://www.boost.org
-Packager: Rene Rivera <gr...@redshift-software.com>
-BuildRoot: /var/tmp/%{name}-%{version}.root
-
-%description
-Boost Jam is a build tool based on FTJam, which in turn is based on
-Perforce Jam. It contains significant improvements made to facilitate
-its use in the Boost Build System, but should be backward compatible
-with Perforce Jam.
-
-Authors:
- Perforce Jam : Cristopher Seiwald
- FT Jam : David Turner
- Boost Jam : David Abrahams
-
-Copyright:
- /+\
- +\ Copyright 1993-2002 Christopher Seiwald and Perforce Software, Inc.
- \+/
- License is hereby granted to use this software and distribute it
- freely, as long as this copyright notice is retained and modifications
- are clearly marked.
- ALL WARRANTIES ARE HEREBY DISCLAIMED.
-
-Also:
- Copyright 2001-2006 David Abrahams.
- Copyright 2002-2006 Rene Rivera.
- Copyright 2003-2006 Vladimir Prus.
-
- Distributed under the Boost Software License, Version 1.0.
- (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-%prep
-%setup -n %{name}-%{version}
-
-%build
-LOCATE_TARGET=bin ./build.sh $BOOST_JAM_TOOLSET
-
-%install
-rm -rf $RPM_BUILD_ROOT
-mkdir -p $RPM_BUILD_ROOT%{_bindir}
-mkdir -p $RPM_BUILD_ROOT%{_docdir}/%{name}-%{version}
-install -m 755 bin/bjam $RPM_BUILD_ROOT%{_bindir}/bjam-%{version}
-ln -sf bjam-%{version} $RPM_BUILD_ROOT%{_bindir}/bjam
-cp -R *.html *.png *.css LICENSE*.txt images jam $RPM_BUILD_ROOT%{_docdir}/%{name}-%{version}
-
-find $RPM_BUILD_ROOT -name CVS -type d -exec rm -r {} \;
-
-%files
-%defattr(-,root,root)
-%attr(755,root,root) /usr/bin/*
-%doc %{_docdir}/%{name}-%{version}
-
-
-%clean
-rm -rf $RPM_BUILD_ROOT
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/boost-no-inspect
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/boost-no-inspect b/ext/kenlm/jam-files/engine/boost-no-inspect
deleted file mode 100644
index 8a06f3a..0000000
--- a/ext/kenlm/jam-files/engine/boost-no-inspect
+++ /dev/null
@@ -1 +0,0 @@
-this really out of our hands, so tell inspect to ignore directory
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/build.bat
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/build.bat b/ext/kenlm/jam-files/engine/build.bat
deleted file mode 100644
index d435c4e..0000000
--- a/ext/kenlm/jam-files/engine/build.bat
+++ /dev/null
@@ -1,590 +0,0 @@
-@ECHO OFF
-
-REM ~ Copyright 2002-2007 Rene Rivera.
-REM ~ Distributed under the Boost Software License, Version 1.0.
-REM ~ (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-setlocal
-goto Start
-
-
-:Set_Error
-color 00
-goto :eof
-
-
-:Clear_Error
-ver >nul
-goto :eof
-
-
-:Error_Print
-REM Output an error message and set the errorlevel to indicate failure.
-setlocal
-ECHO ###
-ECHO ### %1
-ECHO ###
-ECHO ### You can specify the toolset as the argument, i.e.:
-ECHO ### .\build.bat msvc
-ECHO ###
-ECHO ### Toolsets supported by this script are: borland, como, gcc, gcc-nocygwin,
-ECHO ### intel-win32, metrowerks, mingw, msvc, vc7, vc8, vc9, vc10, vc11, vc12
-ECHO ###
-call :Set_Error
-endlocal
-goto :eof
-
-
-:Test_Path
-REM Tests for the given file(executable) presence in the directories in the PATH
-REM environment variable. Additionaly sets FOUND_PATH to the path of the
-REM found file.
-call :Clear_Error
-setlocal
-set test=%~$PATH:1
-endlocal
-if not errorlevel 1 set FOUND_PATH=%~dp$PATH:1
-goto :eof
-
-
-:Test_Option
-REM Tests whether the given string is in the form of an option: "--*"
-call :Clear_Error
-setlocal
-set test=%1
-if not defined test (
- call :Set_Error
- goto Test_Option_End
-)
-set test=###%test%###
-set test=%test:"###=%
-set test=%test:###"=%
-set test=%test:###=%
-if not "-" == "%test:~1,1%" call :Set_Error
-:Test_Option_End
-endlocal
-goto :eof
-
-
-:Test_Empty
-REM Tests whether the given string is not empty
-call :Clear_Error
-setlocal
-set test=%1
-if not defined test (
- call :Clear_Error
- goto Test_Empty_End
-)
-set test=###%test%###
-set test=%test:"###=%
-set test=%test:###"=%
-set test=%test:###=%
-if not "" == "%test%" call :Set_Error
-:Test_Empty_End
-endlocal
-goto :eof
-
-
-:Call_If_Exists
-if EXIST %1 call %*
-goto :eof
-
-
-:Guess_Toolset
-REM Try and guess the toolset to bootstrap the build with...
-REM Sets BOOST_JAM_TOOLSET to the first found toolset.
-REM May also set BOOST_JAM_TOOLSET_ROOT to the
-REM location of the found toolset.
-
-call :Clear_Error
-call :Test_Empty %ProgramFiles%
-if not errorlevel 1 set ProgramFiles=C:\Program Files
-
-call :Clear_Error
-if NOT "_%VS120COMNTOOLS%_" == "__" (
- set "BOOST_JAM_TOOLSET=vc12"
- set "BOOST_JAM_TOOLSET_ROOT=%VS120COMNTOOLS%..\..\VC\"
- goto :eof)
-call :Clear_Error
-if EXIST "%ProgramFiles%\Microsoft Visual Studio 12.0\VC\VCVARSALL.BAT" (
- set "BOOST_JAM_TOOLSET=vc12"
- set "BOOST_JAM_TOOLSET_ROOT=%ProgramFiles%\Microsoft Visual Studio 12.0\VC\"
- goto :eof)
-call :Clear_Error
-if NOT "_%VS110COMNTOOLS%_" == "__" (
- set "BOOST_JAM_TOOLSET=vc11"
- set "BOOST_JAM_TOOLSET_ROOT=%VS110COMNTOOLS%..\..\VC\"
- goto :eof)
-call :Clear_Error
-if EXIST "%ProgramFiles%\Microsoft Visual Studio 11.0\VC\VCVARSALL.BAT" (
- set "BOOST_JAM_TOOLSET=vc11"
- set "BOOST_JAM_TOOLSET_ROOT=%ProgramFiles%\Microsoft Visual Studio 11.0\VC\"
- goto :eof)
-call :Clear_Error
-if NOT "_%VS100COMNTOOLS%_" == "__" (
- set "BOOST_JAM_TOOLSET=vc10"
- set "BOOST_JAM_TOOLSET_ROOT=%VS100COMNTOOLS%..\..\VC\"
- goto :eof)
-call :Clear_Error
-if EXIST "%ProgramFiles%\Microsoft Visual Studio 10.0\VC\VCVARSALL.BAT" (
- set "BOOST_JAM_TOOLSET=vc10"
- set "BOOST_JAM_TOOLSET_ROOT=%ProgramFiles%\Microsoft Visual Studio 10.0\VC\"
- goto :eof)
-call :Clear_Error
-if NOT "_%VS90COMNTOOLS%_" == "__" (
- set "BOOST_JAM_TOOLSET=vc9"
- set "BOOST_JAM_TOOLSET_ROOT=%VS90COMNTOOLS%..\..\VC\"
- goto :eof)
-call :Clear_Error
-if EXIST "%ProgramFiles%\Microsoft Visual Studio 9.0\VC\VCVARSALL.BAT" (
- set "BOOST_JAM_TOOLSET=vc9"
- set "BOOST_JAM_TOOLSET_ROOT=%ProgramFiles%\Microsoft Visual Studio 9.0\VC\"
- goto :eof)
-call :Clear_Error
-if NOT "_%VS80COMNTOOLS%_" == "__" (
- set "BOOST_JAM_TOOLSET=vc8"
- set "BOOST_JAM_TOOLSET_ROOT=%VS80COMNTOOLS%..\..\VC\"
- goto :eof)
-call :Clear_Error
-if EXIST "%ProgramFiles%\Microsoft Visual Studio 8\VC\VCVARSALL.BAT" (
- set "BOOST_JAM_TOOLSET=vc8"
- set "BOOST_JAM_TOOLSET_ROOT=%ProgramFiles%\Microsoft Visual Studio 8\VC\"
- goto :eof)
-call :Clear_Error
-if NOT "_%VS71COMNTOOLS%_" == "__" (
- set "BOOST_JAM_TOOLSET=vc7"
- set "BOOST_JAM_TOOLSET_ROOT=%VS71COMNTOOLS%\..\..\VC7\"
- goto :eof)
-call :Clear_Error
-if NOT "_%VCINSTALLDIR%_" == "__" (
- REM %VCINSTALLDIR% is also set for VC9 (and probably VC8)
- set "BOOST_JAM_TOOLSET=vc7"
- set "BOOST_JAM_TOOLSET_ROOT=%VCINSTALLDIR%\VC7\"
- goto :eof)
-call :Clear_Error
-if EXIST "%ProgramFiles%\Microsoft Visual Studio .NET 2003\VC7\bin\VCVARS32.BAT" (
- set "BOOST_JAM_TOOLSET=vc7"
- set "BOOST_JAM_TOOLSET_ROOT=%ProgramFiles%\Microsoft Visual Studio .NET 2003\VC7\"
- goto :eof)
-call :Clear_Error
-if EXIST "%ProgramFiles%\Microsoft Visual Studio .NET\VC7\bin\VCVARS32.BAT" (
- set "BOOST_JAM_TOOLSET=vc7"
- set "BOOST_JAM_TOOLSET_ROOT=%ProgramFiles%\Microsoft Visual Studio .NET\VC7\"
- goto :eof)
-call :Clear_Error
-if NOT "_%MSVCDir%_" == "__" (
- set "BOOST_JAM_TOOLSET=msvc"
- set "BOOST_JAM_TOOLSET_ROOT=%MSVCDir%\"
- goto :eof)
-call :Clear_Error
-if EXIST "%ProgramFiles%\Microsoft Visual Studio\VC98\bin\VCVARS32.BAT" (
- set "BOOST_JAM_TOOLSET=msvc"
- set "BOOST_JAM_TOOLSET_ROOT=%ProgramFiles%\Microsoft Visual Studio\VC98\"
- goto :eof)
-call :Clear_Error
-if EXIST "%ProgramFiles%\Microsoft Visual C++\VC98\bin\VCVARS32.BAT" (
- set "BOOST_JAM_TOOLSET=msvc"
- set "BOOST_JAM_TOOLSET_ROOT=%ProgramFiles%\Microsoft Visual C++\VC98\"
- goto :eof)
-call :Clear_Error
-call :Test_Path cl.exe
-if not errorlevel 1 (
- set "BOOST_JAM_TOOLSET=msvc"
- set "BOOST_JAM_TOOLSET_ROOT=%FOUND_PATH%..\"
- goto :eof)
-call :Clear_Error
-call :Test_Path vcvars32.bat
-if not errorlevel 1 (
- set "BOOST_JAM_TOOLSET=msvc"
- call "%FOUND_PATH%VCVARS32.BAT"
- set "BOOST_JAM_TOOLSET_ROOT=%MSVCDir%\"
- goto :eof)
-call :Clear_Error
-if EXIST "C:\Borland\BCC55\Bin\bcc32.exe" (
- set "BOOST_JAM_TOOLSET=borland"
- set "BOOST_JAM_TOOLSET_ROOT=C:\Borland\BCC55\"
- goto :eof)
-call :Clear_Error
-call :Test_Path bcc32.exe
-if not errorlevel 1 (
- set "BOOST_JAM_TOOLSET=borland"
- set "BOOST_JAM_TOOLSET_ROOT=%FOUND_PATH%..\"
- goto :eof)
-call :Clear_Error
-call :Test_Path icl.exe
-if not errorlevel 1 (
- set "BOOST_JAM_TOOLSET=intel-win32"
- set "BOOST_JAM_TOOLSET_ROOT=%FOUND_PATH%..\"
- goto :eof)
-call :Clear_Error
-if EXIST "C:\MinGW\bin\gcc.exe" (
- set "BOOST_JAM_TOOLSET=mingw"
- set "BOOST_JAM_TOOLSET_ROOT=C:\MinGW\"
- goto :eof)
-call :Clear_Error
-if NOT "_%CWFolder%_" == "__" (
- set "BOOST_JAM_TOOLSET=metrowerks"
- set "BOOST_JAM_TOOLSET_ROOT=%CWFolder%\"
- goto :eof )
-call :Clear_Error
-call :Test_Path mwcc.exe
-if not errorlevel 1 (
- set "BOOST_JAM_TOOLSET=metrowerks"
- set "BOOST_JAM_TOOLSET_ROOT=%FOUND_PATH%..\..\"
- goto :eof)
-call :Clear_Error
-call :Error_Print "Could not find a suitable toolset."
-goto :eof
-
-
-:Guess_Yacc
-REM Tries to find bison or yacc in common places so we can build the grammar.
-call :Clear_Error
-call :Test_Path yacc.exe
-if not errorlevel 1 (
- set "YACC=yacc -d"
- goto :eof)
-call :Clear_Error
-call :Test_Path bison.exe
-if not errorlevel 1 (
- set "YACC=bison -d --yacc"
- goto :eof)
-call :Clear_Error
-if EXIST "C:\Program Files\GnuWin32\bin\bison.exe" (
- set "YACC=C:\Program Files\GnuWin32\bin\bison.exe" -d --yacc
- goto :eof)
-call :Clear_Error
-call :Error_Print "Could not find Yacc to build the Jam grammar."
-goto :eof
-
-
-:Start
-set BOOST_JAM_TOOLSET=
-set BOOST_JAM_ARGS=
-
-REM If no arguments guess the toolset;
-REM or if first argument is an option guess the toolset;
-REM otherwise the argument is the toolset to use.
-call :Clear_Error
-call :Test_Empty %1
-if not errorlevel 1 (
- call :Guess_Toolset
- if not errorlevel 1 ( goto Setup_Toolset ) else ( goto Finish )
-)
-
-call :Clear_Error
-call :Test_Option %1
-if not errorlevel 1 (
- call :Guess_Toolset
- if not errorlevel 1 ( goto Setup_Toolset ) else ( goto Finish )
-)
-
-call :Clear_Error
-set BOOST_JAM_TOOLSET=%1
-shift
-goto Setup_Toolset
-
-
-:Setup_Toolset
-REM Setup the toolset command and options. This bit of code
-REM needs to be flexible enough to handle both when
-REM the toolset was guessed at and found, or when the toolset
-REM was indicated in the command arguments.
-REM NOTE: The strange multiple "if ?? == _toolset_" tests are that way
-REM because in BAT variables are subsituted only once during a single
-REM command. A complete "if ... ( commands ) else ( commands )"
-REM is a single command, even though it's in multiple lines here.
-:Setup_Args
-call :Clear_Error
-call :Test_Empty %1
-if not errorlevel 1 goto Config_Toolset
-call :Clear_Error
-call :Test_Option %1
-if errorlevel 1 (
- set BOOST_JAM_ARGS=%BOOST_JAM_ARGS% %1
- shift
- goto Setup_Args
-)
-:Config_Toolset
-if NOT "_%BOOST_JAM_TOOLSET%_" == "_metrowerks_" goto Skip_METROWERKS
-if NOT "_%CWFolder%_" == "__" (
- set "BOOST_JAM_TOOLSET_ROOT=%CWFolder%\"
- )
-set "PATH=%BOOST_JAM_TOOLSET_ROOT%Other Metrowerks Tools\Command Line Tools;%PATH%"
-set "BOOST_JAM_CC=mwcc -runtime ss -cwd include -DNT -lkernel32.lib -ladvapi32.lib -luser32.lib"
-set "BOOST_JAM_OPT_JAM=-o bootstrap\jam0.exe"
-set "BOOST_JAM_OPT_MKJAMBASE=-o bootstrap\mkjambase0.exe"
-set "BOOST_JAM_OPT_YYACC=-o bootstrap\yyacc0.exe"
-set "_known_=1"
-:Skip_METROWERKS
-if NOT "_%BOOST_JAM_TOOLSET%_" == "_msvc_" goto Skip_MSVC
-if NOT "_%MSVCDir%_" == "__" (
- set "BOOST_JAM_TOOLSET_ROOT=%MSVCDir%\"
- )
-call :Call_If_Exists "%BOOST_JAM_TOOLSET_ROOT%bin\VCVARS32.BAT"
-if not "_%BOOST_JAM_TOOLSET_ROOT%_" == "__" (
- set "PATH=%BOOST_JAM_TOOLSET_ROOT%bin;%PATH%"
- )
-set "BOOST_JAM_CC=cl /nologo /GZ /Zi /MLd /Fobootstrap/ /Fdbootstrap/ -DNT -DYYDEBUG kernel32.lib advapi32.lib user32.lib"
-set "BOOST_JAM_OPT_JAM=/Febootstrap\jam0"
-set "BOOST_JAM_OPT_MKJAMBASE=/Febootstrap\mkjambase0"
-set "BOOST_JAM_OPT_YYACC=/Febootstrap\yyacc0"
-set "_known_=1"
-:Skip_MSVC
-if NOT "_%BOOST_JAM_TOOLSET%_" == "_vc7_" goto Skip_VC7
-if NOT "_%VS71COMNTOOLS%_" == "__" (
- set "BOOST_JAM_TOOLSET_ROOT=%VS71COMNTOOLS%..\..\VC7\"
- )
-if "_%VCINSTALLDIR%_" == "__" call :Call_If_Exists "%BOOST_JAM_TOOLSET_ROOT%bin\VCVARS32.BAT"
-if NOT "_%BOOST_JAM_TOOLSET_ROOT%_" == "__" (
- if "_%VCINSTALLDIR%_" == "__" (
- set "PATH=%BOOST_JAM_TOOLSET_ROOT%bin;%PATH%"
- ) )
-set "BOOST_JAM_CC=cl /nologo /GZ /Zi /MLd /Fobootstrap/ /Fdbootstrap/ -DNT -DYYDEBUG kernel32.lib advapi32.lib user32.lib"
-set "BOOST_JAM_OPT_JAM=/Febootstrap\jam0"
-set "BOOST_JAM_OPT_MKJAMBASE=/Febootstrap\mkjambase0"
-set "BOOST_JAM_OPT_YYACC=/Febootstrap\yyacc0"
-set "_known_=1"
-:Skip_VC7
-if NOT "_%BOOST_JAM_TOOLSET%_" == "_vc8_" goto Skip_VC8
-if NOT "_%VS80COMNTOOLS%_" == "__" (
- set "BOOST_JAM_TOOLSET_ROOT=%VS80COMNTOOLS%..\..\VC\"
- )
-if "_%VCINSTALLDIR%_" == "__" call :Call_If_Exists "%BOOST_JAM_TOOLSET_ROOT%VCVARSALL.BAT" %BOOST_JAM_ARGS%
-if NOT "_%BOOST_JAM_TOOLSET_ROOT%_" == "__" (
- if "_%VCINSTALLDIR%_" == "__" (
- set "PATH=%BOOST_JAM_TOOLSET_ROOT%bin;%PATH%"
- ) )
-set "BOOST_JAM_CC=cl /nologo /RTC1 /Zi /MTd /Fobootstrap/ /Fdbootstrap/ -DNT -DYYDEBUG -wd4996 kernel32.lib advapi32.lib user32.lib"
-set "BOOST_JAM_OPT_JAM=/Febootstrap\jam0"
-set "BOOST_JAM_OPT_MKJAMBASE=/Febootstrap\mkjambase0"
-set "BOOST_JAM_OPT_YYACC=/Febootstrap\yyacc0"
-set "_known_=1"
-:Skip_VC8
-if NOT "_%BOOST_JAM_TOOLSET%_" == "_vc9_" goto Skip_VC9
-if NOT "_%VS90COMNTOOLS%_" == "__" (
- set "BOOST_JAM_TOOLSET_ROOT=%VS90COMNTOOLS%..\..\VC\"
- )
-if "_%VCINSTALLDIR%_" == "__" call :Call_If_Exists "%BOOST_JAM_TOOLSET_ROOT%VCVARSALL.BAT" %BOOST_JAM_ARGS%
-if NOT "_%BOOST_JAM_TOOLSET_ROOT%_" == "__" (
- if "_%VCINSTALLDIR%_" == "__" (
- set "PATH=%BOOST_JAM_TOOLSET_ROOT%bin;%PATH%"
- ) )
-set "BOOST_JAM_CC=cl /nologo /RTC1 /Zi /MTd /Fobootstrap/ /Fdbootstrap/ -DNT -DYYDEBUG -wd4996 kernel32.lib advapi32.lib user32.lib"
-set "BOOST_JAM_OPT_JAM=/Febootstrap\jam0"
-set "BOOST_JAM_OPT_MKJAMBASE=/Febootstrap\mkjambase0"
-set "BOOST_JAM_OPT_YYACC=/Febootstrap\yyacc0"
-set "_known_=1"
-:Skip_VC9
-if NOT "_%BOOST_JAM_TOOLSET%_" == "_vc10_" goto Skip_VC10
-if NOT "_%VS100COMNTOOLS%_" == "__" (
- set "BOOST_JAM_TOOLSET_ROOT=%VS100COMNTOOLS%..\..\VC\"
- )
-if "_%VCINSTALLDIR%_" == "__" call :Call_If_Exists "%BOOST_JAM_TOOLSET_ROOT%VCVARSALL.BAT" %BOOST_JAM_ARGS%
-if NOT "_%BOOST_JAM_TOOLSET_ROOT%_" == "__" (
- if "_%VCINSTALLDIR%_" == "__" (
- set "PATH=%BOOST_JAM_TOOLSET_ROOT%bin;%PATH%"
- ) )
-set "BOOST_JAM_CC=cl /nologo /RTC1 /Zi /MTd /Fobootstrap/ /Fdbootstrap/ -DNT -DYYDEBUG -wd4996 kernel32.lib advapi32.lib user32.lib"
-set "BOOST_JAM_OPT_JAM=/Febootstrap\jam0"
-set "BOOST_JAM_OPT_MKJAMBASE=/Febootstrap\mkjambase0"
-set "BOOST_JAM_OPT_YYACC=/Febootstrap\yyacc0"
-set "_known_=1"
-:Skip_VC10
-if NOT "_%BOOST_JAM_TOOLSET%_" == "_vc11_" goto Skip_VC11
-if NOT "_%VS110COMNTOOLS%_" == "__" (
- set "BOOST_JAM_TOOLSET_ROOT=%VS110COMNTOOLS%..\..\VC\"
- )
-if "_%VCINSTALLDIR%_" == "__" call :Call_If_Exists "%BOOST_JAM_TOOLSET_ROOT%VCVARSALL.BAT" %BOOST_JAM_ARGS%
-if NOT "_%BOOST_JAM_TOOLSET_ROOT%_" == "__" (
- if "_%VCINSTALLDIR%_" == "__" (
- set "PATH=%BOOST_JAM_TOOLSET_ROOT%bin;%PATH%"
- ) )
-set "BOOST_JAM_CC=cl /nologo /RTC1 /Zi /MTd /Fobootstrap/ /Fdbootstrap/ -DNT -DYYDEBUG -wd4996 kernel32.lib advapi32.lib user32.lib"
-set "BOOST_JAM_OPT_JAM=/Febootstrap\jam0"
-set "BOOST_JAM_OPT_MKJAMBASE=/Febootstrap\mkjambase0"
-set "BOOST_JAM_OPT_YYACC=/Febootstrap\yyacc0"
-set "_known_=1"
-:Skip_VC11
-if NOT "_%BOOST_JAM_TOOLSET%_" == "_vc12_" goto Skip_VC12
-if NOT "_%VS120COMNTOOLS%_" == "__" (
- set "BOOST_JAM_TOOLSET_ROOT=%VS120COMNTOOLS%..\..\VC\"
- )
-if "_%VCINSTALLDIR%_" == "__" call :Call_If_Exists "%BOOST_JAM_TOOLSET_ROOT%VCVARSALL.BAT" %BOOST_JAM_ARGS%
-if NOT "_%BOOST_JAM_TOOLSET_ROOT%_" == "__" (
- if "_%VCINSTALLDIR%_" == "__" (
- set "PATH=%BOOST_JAM_TOOLSET_ROOT%bin;%PATH%"
- ) )
-set "BOOST_JAM_CC=cl /nologo /RTC1 /Zi /MTd /Fobootstrap/ /Fdbootstrap/ -DNT -DYYDEBUG -wd4996 kernel32.lib advapi32.lib user32.lib"
-set "BOOST_JAM_OPT_JAM=/Febootstrap\jam0"
-set "BOOST_JAM_OPT_MKJAMBASE=/Febootstrap\mkjambase0"
-set "BOOST_JAM_OPT_YYACC=/Febootstrap\yyacc0"
-set "_known_=1"
-:Skip_VC12
-if NOT "_%BOOST_JAM_TOOLSET%_" == "_borland_" goto Skip_BORLAND
-if "_%BOOST_JAM_TOOLSET_ROOT%_" == "__" (
- call :Test_Path bcc32.exe )
-if "_%BOOST_JAM_TOOLSET_ROOT%_" == "__" (
- if not errorlevel 1 (
- set "BOOST_JAM_TOOLSET_ROOT=%FOUND_PATH%..\"
- ) )
-if not "_%BOOST_JAM_TOOLSET_ROOT%_" == "__" (
- set "PATH=%BOOST_JAM_TOOLSET_ROOT%Bin;%PATH%"
- )
-set "BOOST_JAM_CC=bcc32 -WC -w- -q -I%BOOST_JAM_TOOLSET_ROOT%Include -L%BOOST_JAM_TOOLSET_ROOT%Lib /DNT -nbootstrap"
-set "BOOST_JAM_OPT_JAM=-ejam0"
-set "BOOST_JAM_OPT_MKJAMBASE=-emkjambasejam0"
-set "BOOST_JAM_OPT_YYACC=-eyyacc0"
-set "_known_=1"
-:Skip_BORLAND
-if NOT "_%BOOST_JAM_TOOLSET%_" == "_como_" goto Skip_COMO
-set "BOOST_JAM_CC=como -DNT"
-set "BOOST_JAM_OPT_JAM=-o bootstrap\jam0.exe"
-set "BOOST_JAM_OPT_MKJAMBASE=-o bootstrap\mkjambase0.exe"
-set "BOOST_JAM_OPT_YYACC=-o bootstrap\yyacc0.exe"
-set "_known_=1"
-:Skip_COMO
-if NOT "_%BOOST_JAM_TOOLSET%_" == "_gcc_" goto Skip_GCC
-set "BOOST_JAM_CC=gcc -DNT"
-set "BOOST_JAM_OPT_JAM=-o bootstrap\jam0.exe"
-set "BOOST_JAM_OPT_MKJAMBASE=-o bootstrap\mkjambase0.exe"
-set "BOOST_JAM_OPT_YYACC=-o bootstrap\yyacc0.exe"
-set "_known_=1"
-:Skip_GCC
-if NOT "_%BOOST_JAM_TOOLSET%_" == "_gcc-nocygwin_" goto Skip_GCC_NOCYGWIN
-set "BOOST_JAM_CC=gcc -DNT -mno-cygwin"
-set "BOOST_JAM_OPT_JAM=-o bootstrap\jam0.exe"
-set "BOOST_JAM_OPT_MKJAMBASE=-o bootstrap\mkjambase0.exe"
-set "BOOST_JAM_OPT_YYACC=-o bootstrap\yyacc0.exe"
-set "_known_=1"
-:Skip_GCC_NOCYGWIN
-if NOT "_%BOOST_JAM_TOOLSET%_" == "_intel-win32_" goto Skip_INTEL_WIN32
-set "BOOST_JAM_CC=icl -DNT /nologo kernel32.lib advapi32.lib user32.lib"
-set "BOOST_JAM_OPT_JAM=/Febootstrap\jam0"
-set "BOOST_JAM_OPT_MKJAMBASE=/Febootstrap\mkjambase0"
-set "BOOST_JAM_OPT_YYACC=/Febootstrap\yyacc0"
-set "_known_=1"
-:Skip_INTEL_WIN32
-if NOT "_%BOOST_JAM_TOOLSET%_" == "_mingw_" goto Skip_MINGW
-if not "_%BOOST_JAM_TOOLSET_ROOT%_" == "__" (
- set "PATH=%BOOST_JAM_TOOLSET_ROOT%bin;%PATH%"
- )
-set "BOOST_JAM_CC=gcc -DNT"
-set "BOOST_JAM_OPT_JAM=-o bootstrap\jam0.exe"
-set "BOOST_JAM_OPT_MKJAMBASE=-o bootstrap\mkjambase0.exe"
-set "BOOST_JAM_OPT_YYACC=-o bootstrap\yyacc0.exe"
-set "_known_=1"
-:Skip_MINGW
-call :Clear_Error
-if "_%_known_%_" == "__" (
- call :Error_Print "Unknown toolset: %BOOST_JAM_TOOLSET%"
-)
-if errorlevel 1 goto Finish
-
-echo ###
-echo ### Using '%BOOST_JAM_TOOLSET%' toolset.
-echo ###
-
-set YYACC_SOURCES=yyacc.c
-set MKJAMBASE_SOURCES=mkjambase.c
-set BJAM_SOURCES=
-set BJAM_SOURCES=%BJAM_SOURCES% command.c compile.c constants.c debug.c
-set BJAM_SOURCES=%BJAM_SOURCES% execcmd.c execnt.c filent.c frames.c function.c
-set BJAM_SOURCES=%BJAM_SOURCES% glob.c hash.c hdrmacro.c headers.c jam.c
-set BJAM_SOURCES=%BJAM_SOURCES% jambase.c jamgram.c lists.c make.c make1.c
-set BJAM_SOURCES=%BJAM_SOURCES% object.c option.c output.c parse.c pathnt.c
-set BJAM_SOURCES=%BJAM_SOURCES% pathsys.c regexp.c rules.c scan.c search.c
-set BJAM_SOURCES=%BJAM_SOURCES% subst.c timestamp.c variable.c modules.c
-set BJAM_SOURCES=%BJAM_SOURCES% strings.c filesys.c builtins.c md5.c class.c
-set BJAM_SOURCES=%BJAM_SOURCES% cwd.c w32_getreg.c native.c modules/set.c
-set BJAM_SOURCES=%BJAM_SOURCES% modules/path.c modules/regex.c
-set BJAM_SOURCES=%BJAM_SOURCES% modules/property-set.c modules/sequence.c
-set BJAM_SOURCES=%BJAM_SOURCES% modules/order.c
-
-set BJAM_UPDATE=
-:Check_Update
-call :Test_Empty %1
-if not errorlevel 1 goto Check_Update_End
-call :Clear_Error
-setlocal
-set test=%1
-set test=###%test%###
-set test=%test:"###=%
-set test=%test:###"=%
-set test=%test:###=%
-if "%test%" == "--update" goto Found_Update
-endlocal
-shift
-if not "_%BJAM_UPDATE%_" == "_update_" goto Check_Update
-:Found_Update
-endlocal
-set BJAM_UPDATE=update
-:Check_Update_End
-if "_%BJAM_UPDATE%_" == "_update_" (
- if not exist ".\bootstrap\jam0.exe" (
- set BJAM_UPDATE=
- )
-)
-
-@echo ON
-@if "_%BJAM_UPDATE%_" == "_update_" goto Skip_Bootstrap
-if exist bootstrap rd /S /Q bootstrap
-md bootstrap
-@if not exist jamgram.y goto Bootstrap_GrammarPrep
-@if not exist jamgramtab.h goto Bootstrap_GrammarPrep
-@goto Skip_GrammarPrep
-:Bootstrap_GrammarPrep
-%BOOST_JAM_CC% %BOOST_JAM_OPT_YYACC% %YYACC_SOURCES%
-@if not exist ".\bootstrap\yyacc0.exe" goto Skip_GrammarPrep
-.\bootstrap\yyacc0 jamgram.y jamgramtab.h jamgram.yy
-:Skip_GrammarPrep
-@if not exist jamgram.c goto Bootstrap_GrammarBuild
-@if not exist jamgram.h goto Bootstrap_GrammarBuild
-@goto Skip_GrammarBuild
-:Bootstrap_GrammarBuild
-@echo OFF
-if "_%YACC%_" == "__" (
- call :Guess_Yacc
-)
-if errorlevel 1 goto Finish
-@echo ON
-%YACC% jamgram.y
-@if errorlevel 1 goto Finish
-del /f jamgram.c
-rename y.tab.c jamgram.c
-del /f jamgram.h
-rename y.tab.h jamgram.h
-:Skip_GrammarBuild
-@echo ON
-@if exist jambase.c goto Skip_Jambase
-%BOOST_JAM_CC% %BOOST_JAM_OPT_MKJAMBASE% %MKJAMBASE_SOURCES%
-@if not exist ".\bootstrap\mkjambase0.exe" goto Skip_Jambase
-.\bootstrap\mkjambase0 jambase.c Jambase
-:Skip_Jambase
-%BOOST_JAM_CC% %BOOST_JAM_OPT_JAM% %BJAM_SOURCES%
-:Skip_Bootstrap
-@if not exist ".\bootstrap\jam0.exe" goto Skip_Jam
-@set args=%*
-@echo OFF
-:Set_Args
-setlocal
-call :Test_Empty %args%
-if not errorlevel 1 goto Set_Args_End
-set test=###%args:~0,2%###
-set test=%test:"###=%
-set test=%test:###"=%
-set test=%test:###=%
-set test=%test:~0,1%
-if "-" == "%test%" goto Set_Args_End
-endlocal
-set args=%args:~1%
-goto Set_Args
-:Set_Args_End
-@echo ON
-@if "_%BJAM_UPDATE%_" == "_update_" goto Skip_Clean
-.\bootstrap\jam0 -f build.jam --toolset=%BOOST_JAM_TOOLSET% "--toolset-root=%BOOST_JAM_TOOLSET_ROOT% " %args% clean
-:Skip_Clean
-.\bootstrap\jam0 -f build.jam --toolset=%BOOST_JAM_TOOLSET% "--toolset-root=%BOOST_JAM_TOOLSET_ROOT% " %args%
-:Skip_Jam
-
-:Finish
[32/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/util/doc.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/util/doc.jam b/ext/kenlm/jam-files/boost-build/util/doc.jam
deleted file mode 100644
index 7ff2df4..0000000
--- a/ext/kenlm/jam-files/boost-build/util/doc.jam
+++ /dev/null
@@ -1,1002 +0,0 @@
-# Copyright 2002, 2005 Dave Abrahams
-# Copyright 2002, 2003, 2006 Rene Rivera
-# Copyright 2003 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-# Documentation system, handles --help requests.
-# It defines rules that attach documentation to modules, rules, and variables.
-# Collects and generates documentation for the various parts of the build
-# system. The documentation is collected from comments integrated into the code.
-
-import modules ;
-import print ;
-import set ;
-import container ;
-import "class" ;
-import sequence ;
-import path ;
-
-
-# The type of output to generate.
-# "console" is formated text echoed to the console (the default);
-# "text" is formated text appended to the output file;
-# "html" is HTML output to the file.
-#
-help-output = console ;
-
-
-# The file to output documentation to when generating "text" or "html" help.
-# This is without extension as the extension is determined by the type of
-# output.
-#
-help-output-file = help ;
-
-# Whether to include local rules in help output.
-#
-.option.show-locals ?= ;
-
-# When showing documentation for a module, whether to also generate
-# automatically the detailed docs for each item in the module.
-#
-.option.detailed ?= ;
-
-# Generate debug output as the help is generated and modules are parsed.
-#
-.option.debug ?= ;
-
-# Enable or disable a documentation option.
-#
-local rule set-option (
- option # The option name.
- : value ? # Enabled (non-empty), or disabled (empty)
-)
-{
- .option.$(option) = $(value) ;
-}
-
-
-# Set the type of output.
-#
-local rule set-output ( type )
-{
- help-output = $(type) ;
-}
-
-
-# Set the output to a file.
-#
-local rule set-output-file ( file )
-{
- help-output-file = $(file) ;
-}
-
-
-# Extracts the brief comment from a complete comment. The brief comment is the
-# first sentence.
-#
-local rule brief-comment (
- docs * # The comment documentation.
-)
-{
- local d = $(docs:J=" ") ;
- local p = [ MATCH ".*([.])$" : $(d) ] ;
- if ! $(p) { d = $(d)"." ; }
- d = $(d)" " ;
- local m = [ MATCH "^([^.]+[.])(.*)" : $(d) ] ;
- local brief = $(m[1]) ;
- while $(m[2]) && [ MATCH "^([^ ])" : $(m[2]) ]
- {
- m = [ MATCH "^([^.]+[.])(.*)" : $(m[2]) ] ;
- brief += $(m[1]) ;
- }
- return $(brief:J="") ;
-}
-
-
-# Specifies the documentation for the current module.
-#
-local rule set-module-doc (
- module-name ? # The name of the module to document.
- : docs * # The documentation for the module.
-)
-{
- module-name ?= * ;
-
- $(module-name).brief = [ brief-comment $(docs) ] ;
- $(module-name).docs = $(docs) ;
-
- if ! $(module-name) in $(documented-modules)
- {
- documented-modules += $(module-name) ;
- }
-}
-
-
-# Specifies the documentation for the current module.
-#
-local rule set-module-copyright (
- module-name ? # The name of the module to document.
- : copyright * # The copyright for the module.
-)
-{
- module-name ?= * ;
-
- $(module-name).copy-brief = [ brief-comment $(copyright) ] ;
- $(module-name).copy-docs = $(docs) ;
-
- if ! $(module-name) in $(documented-modules)
- {
- documented-modules += $(module-name) ;
- }
-}
-
-
-# Specifies the documentation for a rule in the current module. If called in the
-# global module, this documents a global rule.
-#
-local rule set-rule-doc (
- name # The name of the rule.
- module-name ? # The name of the module to document.
- is-local ? # Whether the rule is local to the module.
- : docs * # The documentation for the rule.
-)
-{
- module-name ?= * ;
-
- $(module-name).$(name).brief = [ brief-comment $(docs) ] ;
- $(module-name).$(name).docs = $(docs) ;
- $(module-name).$(name).is-local = $(is-local) ;
-
- if ! $(name) in $($(module-name).rules)
- {
- $(module-name).rules += $(name) ;
- }
-}
-
-
-# Specify a class, will turn a rule into a class.
-#
-local rule set-class-doc (
- name # The name of the class.
- module-name ? # The name of the module to document.
- : super-name ? # The super class name.
-)
-{
- module-name ?= * ;
-
- $(module-name).$(name).is-class = true ;
- $(module-name).$(name).super-name = $(super-name) ;
- $(module-name).$(name).class-rules =
- [ MATCH "^($(name)[.].*)" : $($(module-name).rules) ] ;
- $(module-name).$($(module-name).$(name).class-rules).is-class-rule = true ;
-
- $(module-name).classes += $(name) ;
- $(module-name).class-rules += $($(module-name).$(name).class-rules) ;
- $(module-name).rules =
- [ set.difference $($(module-name).rules) :
- $(name) $($(module-name).$(name).class-rules) ] ;
-}
-
-
-# Set the argument call signature of a rule.
-#
-local rule set-rule-arguments-signature (
- name # The name of the rule.
- module-name ? # The name of the module to document.
- : signature * # The arguments signature.
-)
-{
- module-name ?= * ;
-
- $(module-name).$(name).signature = $(signature) ;
-}
-
-
-# Specifies the documentation for an argument of a rule.
-#
-local rule set-argument-doc (
- name # The name of the argument.
- qualifier # Argument syntax qualifier, "*", "+", etc.
- rule-name # The name of the rule.
- module-name ? # THe optional name of the module.
- : docs * # The documentation.
-)
-{
- module-name ?= * ;
-
- $(module-name).$(rule-name).args.$(name).qualifier = $(qualifier) ;
- $(module-name).$(rule-name).args.$(name).docs = $(docs) ;
-
- if ! $(name) in $($(module-name).$(rule-name).args)
- {
- $(module-name).$(rule-name).args += $(name) ;
- }
-}
-
-
-# Specifies the documentation for a variable in the current module. If called in
-# the global module, the global variable is documented.
-#
-local rule set-variable-doc (
- name # The name of the variable.
- default # The default value.
- initial # The initial value.
- module-name ? # The name of the module to document.
- : docs * # The documentation for the variable.
-)
-{
- module-name ?= * ;
-
- $(module-name).$(name).brief = [ brief-comment $(docs) ] ;
- $(module-name).$(name).default = $(default) ;
- $(module-name).$(name).initial = $(initial) ;
- $(module-name).$(name).docs = $(docs) ;
-
- if ! $(name) in $($(module-name).variables)
- {
- $(module-name).variables += $(name) ;
- }
-}
-
-
-# Generates a general description of the documentation and help system.
-#
-local rule print-help-top ( )
-{
- print.section "General command line usage" ;
-
- print.text " b2 [options] [properties] [targets]
-
- Options, properties and targets can be specified in any order.
- " ;
-
- print.section "Important Options" ;
-
- print.list-start ;
- print.list-item "--clean Remove targets instead of building" ;
- print.list-item "-a Rebuild everything" ;
- print.list-item "-n Don't execute the commands, only print them" ;
- print.list-item "-d+2 Show commands as they are executed" ;
- print.list-item "-d0 Supress all informational messages" ;
- print.list-item "-q Stop at first error" ;
- print.list-item "--reconfigure Rerun all configuration checks" ;
- print.list-item "--debug-configuration Diagnose configuration" ;
- print.list-item "--debug-building Report which targets are built with what properties" ;
- print.list-item "--debug-generator Diagnose generator search/execution" ;
- print.list-end ;
-
- print.section "Further Help"
- The following options can be used to obtain additional documentation.
- ;
-
- print.list-start ;
- print.list-item "--help-options Print more obscure command line options." ;
- print.list-item "--help-internal Boost.Build implementation details." ;
- print.list-item "--help-doc-options Implementation details doc formatting." ;
- print.list-end ;
-}
-
-
-# Generate Jam/Boost.Jam command usage information.
-#
-local rule print-help-usage ( )
-{
- print.section "Boost.Build Usage"
- "b2 [ options... ] targets..."
- ;
- print.list-start ;
- print.list-item -a;
- Build all targets, even if they are current. ;
- print.list-item -fx;
- Read '"x"' as the Jamfile for building instead of searching for the
- Boost.Build system. ;
- print.list-item -jx;
- Run up to '"x"' commands concurrently. ;
- print.list-item -n;
- Do not execute build commands. Instead print out the commands as they
- would be executed if building. ;
- print.list-item -ox;
- Output the used build commands to file '"x"'. ;
- print.list-item -q;
- Quit as soon as a build failure is encountered. Without this option
- Boost.Jam will continue building as many targets as it can. ;
- print.list-item -sx=y;
- Sets a Jam variable '"x"' to the value '"y"', overriding any value that
- variable would have from the environment. ;
- print.list-item -tx;
- Rebuild the target '"x"', even if it is up-to-date. ;
- print.list-item -v;
- Display the version of b2. ;
- print.list-item --x;
- Any option not explicitly handled by Boost.Build remains available to
- build scripts using the '"ARGV"' variable. ;
- print.list-item --abbreviate-paths;
- Use abbreviated paths for targets. ;
- print.list-item --hash;
- Shorten target paths by using an MD5 hash. ;
- print.list-item -dn;
- Enables output of diagnostic messages. The debug level '"n"' and all
- below it are enabled by this option. ;
- print.list-item -d+n;
- Enables output of diagnostic messages. Only the output for debug level
- '"n"' is enabled. ;
- print.list-end ;
- print.section "Debug Levels"
- Each debug level shows a different set of information. Usually with
- higher levels producing more verbose information. The following levels
- are supported: ;
- print.list-start ;
- print.list-item 0;
- Turn off all diagnostic output. Only errors are reported. ;
- print.list-item 1;
- Show the actions taken for building targets, as they are executed. ;
- print.list-item 2;
- Show "quiet" actions and display all action text, as they are executed. ;
- print.list-item 3;
- Show dependency analysis, and target/source timestamps/paths. ;
- print.list-item 4;
- Show arguments of shell invocations. ;
- print.list-item 5;
- Show rule invocations and variable expansions. ;
- print.list-item 6;
- Show directory/header file/archive scans, and attempts at binding to targets. ;
- print.list-item 7;
- Show variable settings. ;
- print.list-item 8;
- Show variable fetches, variable expansions, and evaluation of '"if"' expressions. ;
- print.list-item 9;
- Show variable manipulation, scanner tokens, and memory usage. ;
- print.list-item 10;
- Show execution times for rules. ;
- print.list-item 11;
- Show parsing progress of Jamfiles. ;
- print.list-item 12;
- Show graph for target dependencies. ;
- print.list-item 13;
- Show changes in target status (fate). ;
- print.list-end ;
-}
-
-
-# Generates description of options controlling the help system. This
-# automatically reads the options as all variables in the doc module of the form
-# ".option.*".
-#
-local rule print-help-options (
- module-name # The doc module.
-)
-{
- print.section "Help Options"
- These are all the options available for enabling or disabling to control
- the help system in various ways. Options can be enabled or disabled with
- '"--help-enable-<option>"', and "'--help-disable-<option>'"
- respectively.
- ;
- local options-to-list = [ MATCH ^[.]option[.](.*) : $($(module-name).variables) ] ;
- if $(options-to-list)
- {
- print.list-start ;
- for local option in [ sequence.insertion-sort $(options-to-list) ]
- {
- local def = disabled ;
- if $($(module-name)..option.$(option).default) != "(empty)"
- {
- def = enabled ;
- }
- print.list-item $(option): $($(module-name)..option.$(option).docs)
- Default is $(def). ;
- }
- print.list-end ;
- }
-}
-
-
-# Generate brief documentation for all the known items in the section for a
-# module. Possible sections are: "rules", and "variables".
-#
-local rule print-help-module-section (
- module # The module name.
- section # rules or variables.
- : section-head # The title of the section.
- section-description * # The detailed description of the section.
-)
-{
- if $($(module).$(section))
- {
- print.section $(section-head) $(section-description) ;
- print.list-start ;
- for local item in [ sequence.insertion-sort $($(module).$(section)) ]
- {
- local show = ;
- if ! $($(module).$(item).is-local)
- {
- show = yes ;
- }
- if $(.option.show-locals)
- {
- show = yes ;
- }
- if $(show)
- {
- print.list-item $(item): $($(module).$(item).brief) ;
- }
- }
- print.list-end ;
- }
-}
-
-
-# Generate documentation for all possible modules. We attempt to list all known
-# modules together with a brief description of each.
-#
-local rule print-help-all (
- ignored # Usually the module name, but is ignored here.
-)
-{
- print.section "Modules"
- "These are all the known modules. Use --help <module> to get more"
- "detailed information."
- ;
- if $(documented-modules)
- {
- print.list-start ;
- for local module-name in [ sequence.insertion-sort $(documented-modules) ]
- {
- # The brief docs for each module.
- print.list-item $(module-name): $($(module-name).brief) ;
- }
- print.list-end ;
- }
- # The documentation for each module when details are requested.
- if $(documented-modules) && $(.option.detailed)
- {
- for local module-name in [ sequence.insertion-sort $(documented-modules) ]
- {
- # The brief docs for each module.
- print-help-module $(module-name) ;
- }
- }
-}
-
-
-# Generate documentation for a module. Basic information about the module is
-# generated.
-#
-local rule print-help-module (
- module-name # The module to generate docs for.
-)
-{
- # Print the docs.
- print.section "Module '$(module-name)'" $($(module-name).docs) ;
-
- # Print out the documented classes.
- print-help-module-section $(module-name) classes : "Module '$(module-name)' classes"
- Use --help $(module-name).<class-name> to get more information. ;
-
- # Print out the documented rules.
- print-help-module-section $(module-name) rules : "Module '$(module-name)' rules"
- Use --help $(module-name).<rule-name> to get more information. ;
-
- # Print out the documented variables.
- print-help-module-section $(module-name) variables : "Module '$(module-name)' variables"
- Use --help $(module-name).<variable-name> to get more information. ;
-
- # Print out all the same information but indetailed form.
- if $(.option.detailed)
- {
- print-help-classes $(module-name) ;
- print-help-rules $(module-name) ;
- print-help-variables $(module-name) ;
- }
-}
-
-
-# Generate documentation for a set of rules in a module.
-#
-local rule print-help-rules (
- module-name # Module of the rules.
- : name * # Optional list of rules to describe.
-)
-{
- name ?= $($(module-name).rules) ;
- if [ set.intersection $(name) : $($(module-name).rules) $($(module-name).class-rules) ]
- {
- # Print out the given rules.
- for local rule-name in [ sequence.insertion-sort $(name) ]
- {
- if $(.option.show-locals) || ! $($(module-name).$(rule-name).is-local)
- {
- local signature = $($(module-name).$(rule-name).signature:J=" ") ;
- signature ?= "" ;
- print.section "Rule '$(module-name).$(rule-name) ( $(signature) )'"
- $($(module-name).$(rule-name).docs) ;
- if $($(module-name).$(rule-name).args)
- {
- print.list-start ;
- for local arg-name in $($(module-name).$(rule-name).args)
- {
- print.list-item $(arg-name): $($(module-name).$(rule-name).args.$(arg-name).docs) ;
- }
- print.list-end ;
- }
- }
- }
- }
-}
-
-
-# Generate documentation for a set of classes in a module.
-#
-local rule print-help-classes (
- module-name # Module of the classes.
- : name * # Optional list of classes to describe.
-)
-{
- name ?= $($(module-name).classes) ;
- if [ set.intersection $(name) : $($(module-name).classes) ]
- {
- # Print out the given classes.
- for local class-name in [ sequence.insertion-sort $(name) ]
- {
- if $(.option.show-locals) || ! $($(module-name).$(class-name).is-local)
- {
- local signature = $($(module-name).$(class-name).signature:J=" ") ;
- signature ?= "" ;
- print.section "Class '$(module-name).$(class-name) ( $(signature) )'"
- $($(module-name).$(class-name).docs)
- "Inherits from '"$($(module-name).$(class-name).super-name)"'." ;
- if $($(module-name).$(class-name).args)
- {
- print.list-start ;
- for local arg-name in $($(module-name).$(class-name).args)
- {
- print.list-item $(arg-name): $($(module-name).$(class-name).args.$(arg-name).docs) ;
- }
- print.list-end ;
- }
- }
-
- # Print out the documented rules of the class.
- print-help-module-section $(module-name) $(class-name).class-rules : "Class '$(module-name).$(class-name)' rules"
- Use --help $(module-name).<rule-name> to get more information. ;
-
- # Print out all the rules if details are requested.
- if $(.option.detailed)
- {
- print-help-rules $(module-name) : $($(module-name).$(class-name).class-rules) ;
- }
- }
- }
-}
-
-
-# Generate documentation for a set of variables in a module.
-#
-local rule print-help-variables (
- module-name ? # Module of the variables.
- : name * # Optional list of variables to describe.
-)
-{
- name ?= $($(module-name).variables) ;
- if [ set.intersection $(name) : $($(module-name).variables) ]
- {
- # Print out the given variables.
- for local variable-name in [ sequence.insertion-sort $(name) ]
- {
- print.section "Variable '$(module-name).$(variable-name)'" $($(module-name).$(variable-name).docs) ;
- if $($(module-name).$(variable-name).default) ||
- $($(module-name).$(variable-name).initial)
- {
- print.list-start ;
- if $($(module-name).$(variable-name).default)
- {
- print.list-item "default value:" '$($(module-name).$(variable-name).default:J=" ")' ;
- }
- if $($(module-name).$(variable-name).initial)
- {
- print.list-item "initial value:" '$($(module-name).$(variable-name).initial:J=" ")' ;
- }
- print.list-end ;
- }
- }
- }
-}
-
-
-# Generate documentation for a project.
-#
-local rule print-help-project (
- unused ?
- : jamfile * # The project Jamfile.
-)
-{
- if $(jamfile<$(jamfile)>.docs)
- {
- # Print the docs.
- print.section "Project-specific help"
- Project has jamfile at $(jamfile) ;
-
- print.lines $(jamfile<$(jamfile)>.docs) "" ;
- }
-}
-
-
-# Generate documentation for a config file.
-#
-local rule print-help-config (
- unused ?
- : type # The type of configuration file user or site.
- config-file # The configuration Jamfile.
-)
-{
- if $(jamfile<$(config-file)>.docs)
- {
- # Print the docs.
- print.section "Configuration help"
- Configuration file at $(config-file) ;
-
- print.lines $(jamfile<$(config-file)>.docs) "" ;
- }
-}
-
-
-ws = " " ;
-
-# Extract the text from a block of comments.
-#
-local rule extract-comment (
- var # The name of the variable to extract from.
-)
-{
- local comment = ;
- local line = $($(var)[1]) ;
- local l = [ MATCH "^[$(ws)]*(#)(.*)$" : $(line) ] ;
- while $(l[1]) && $($(var))
- {
- if $(l[2]) { comment += [ MATCH "^[$(ws)]?(.*)$" : $(l[2]) ] ; }
- else { comment += "" ; }
- $(var) = $($(var)[2-]) ;
- line = $($(var)[1]) ;
- l = [ MATCH "^[$(ws)]*(#)(.*)$" : $(line) ] ;
- }
- return $(comment) ;
-}
-
-
-# Extract s single line of Jam syntax, ignoring any comments.
-#
-local rule extract-syntax (
- var # The name of the variable to extract from.
-)
-{
- local syntax = ;
- local line = $($(var)[1]) ;
- while ! $(syntax) && ! [ MATCH "^[$(ws)]*(#)" : $(line) ] && $($(var))
- {
- local m = [ MATCH "^[$(ws)]*(.*)$" : $(line) ] ;
- if $(m)
- {
- syntax = $(m) ;
- }
- $(var) = $($(var)[2-]) ;
- line = $($(var)[1]) ;
- }
- return $(syntax) ;
-}
-
-
-# Extract the next token, this is either a single Jam construct or a comment as
-# a single token.
-#
-local rule extract-token (
- var # The name of the variable to extract from.
-)
-{
- local parts = ;
- while ! $(parts)
- {
- parts = [ MATCH "^[$(ws)]*([^$(ws)]+)[$(ws)]*(.*)" : $($(var)[1]) ] ;
- if ! $(parts)
- {
- $(var) = $($(var)[2-]) ;
- }
- }
- local token = ;
- if [ MATCH "^(#)" : $(parts[1]) ]
- {
- token = $(parts:J=" ") ;
- $(var) = $($(var)[2-]) ;
- }
- else
- {
- token = $(parts[1]) ;
- $(var) = $(parts[2-]:J=" ") $($(var)[2-]) ;
- }
- return $(token) ;
-}
-
-
-# Scan for a rule declaration as the next item in the variable.
-#
-local rule scan-rule (
- syntax ? # The first part of the text which contains the rule declaration.
- : var # The name of the variable to extract from.
-)
-{
- local rule-parts =
- [ MATCH "^[$(ws)]*(rule|local[$(ws)]*rule)[$(ws)]+([^$(ws)]+)[$(ws)]*(.*)" : $(syntax:J=" ") ] ;
- if $(rule-parts[1])
- {
- # Mark as doc for rule.
- local rule-name = $(rule-parts[2]) ;
- if $(scope-name)
- {
- rule-name = $(scope-name).$(rule-name) ;
- }
- local is-local = [ MATCH "^(local).*" : $(rule-parts[1]) ] ;
- if $(comment-block)
- {
- set-rule-doc $(rule-name) $(module-name) $(is-local) : $(comment-block) ;
- }
- # Parse args of rule.
- $(var) = $(rule-parts[3-]) $($(var)) ;
- set-rule-arguments-signature $(rule-name) $(module-name) : [ scan-rule-arguments $(var) ] ;
- # Scan within this rules scope.
- local scope-level = [ extract-token $(var) ] ;
- local scope-name = $(rule-name) ;
- while $(scope-level)
- {
- local comment-block = [ extract-comment $(var) ] ;
- local syntax-block = [ extract-syntax $(var) ] ;
- if [ scan-rule $(syntax-block) : $(var) ]
- {
- }
- else if [ MATCH "^(\\{)" : $(syntax-block) ]
- {
- scope-level += "{" ;
- }
- else if [ MATCH "^[^\\}]*([\\}])[$(ws)]*$" : $(syntax-block) ]
- {
- scope-level = $(scope-level[2-]) ;
- }
- }
-
- return true ;
- }
-}
-
-
-# Scan the arguments of a rule.
-#
-local rule scan-rule-arguments (
- var # The name of the variable to extract from.
-)
-{
- local arg-syntax = ;
- local token = [ extract-token $(var) ] ;
- while $(token) != "(" && $(token) != "{"
- {
- token = [ extract-token $(var) ] ;
- }
- if $(token) != "{"
- {
- token = [ extract-token $(var) ] ;
- }
- local arg-signature = ;
- while $(token) != ")" && $(token) != "{"
- {
- local arg-name = ;
- local arg-qualifier = " " ;
- local arg-doc = ;
- if $(token) = ":"
- {
- arg-signature += $(token) ;
- token = [ extract-token $(var) ] ;
- }
- arg-name = $(token) ;
- arg-signature += $(token) ;
- token = [ extract-token $(var) ] ;
- if [ MATCH "^([\\*\\+\\?])" : $(token) ]
- {
- arg-qualifier = $(token) ;
- arg-signature += $(token) ;
- token = [ extract-token $(var) ] ;
- }
- if $(token) = ":"
- {
- arg-signature += $(token) ;
- token = [ extract-token $(var) ] ;
- }
- if [ MATCH "^(#)" : $(token) ]
- {
- $(var) = $(token) $($(var)) ;
- arg-doc = [ extract-comment $(var) ] ;
- token = [ extract-token $(var) ] ;
- }
- set-argument-doc $(arg-name) $(arg-qualifier) $(rule-name) $(module-name) : $(arg-doc) ;
- }
- while $(token) != "{"
- {
- token = [ extract-token $(var) ] ;
- }
- $(var) = "{" $($(var)) ;
- arg-signature ?= "" ;
- return $(arg-signature) ;
-}
-
-
-# Scan for a variable declaration.
-#
-local rule scan-variable (
- syntax ? # The first part of the text which contains the variable declaration.
- : var # The name of the variable to extract from.
-)
-{
- # [1] = name, [2] = value(s)
- local var-parts =
- [ MATCH "^[$(ws)]*([^$(ws)]+)[$(ws)]+([\\?\\=]*)[$(ws)]+([^\\;]*)\\;" : $(syntax) ] ;
- if $(var-parts)
- {
- local value = [ MATCH "^(.*)[ ]$" : $(var-parts[3-]:J=" ") ] ;
- local default-value = "" ;
- local initial-valie = "" ;
- if $(var-parts[2]) = "?="
- {
- default-value = $(value) ;
- default-value ?= "(empty)" ;
- }
- else
- {
- initial-value = $(value) ;
- initial-value ?= "(empty)" ;
- }
- if $(comment-block)
- {
- set-variable-doc $(var-parts[1]) $(default-value) $(initial-value) $(module-name) : $(comment-block) ;
- }
- return true ;
- }
-}
-
-
-# Scan a class declaration.
-#
-local rule scan-class (
- syntax ? # The syntax text for the class declaration.
-)
-{
- # [1] = class?, [2] = name, [3] = superclass
- local class-parts =
- [ MATCH "^[$(ws)]*([^$(ws)]+)[$(ws)]+([^$(ws)]+)[$(ws)]+:*[$(ws)]*([^$(ws);]*)" : $(syntax) ] ;
- if $(class-parts[1]) = "class" || $(class-parts[1]) = "class.class"
- {
- set-class-doc $(class-parts[2]) $(module-name) : $(class-parts[3]) ;
- }
-}
-
-
-# Scan a module file for documentation comments. This also invokes any actions
-# assigned to the module. The actions are the rules that do the actual output of
-# the documentation. This rule is invoked as the header scan rule for the module
-# file.
-#
-rule scan-module (
- target # The module file.
- : text * # The text in the file, one item per line.
- : action * # Rule to call to output docs for the module.
-)
-{
- if $(.option.debug) { ECHO "HELP:" scanning module target '$(target)' ; }
- local module-name = $(target:B) ;
- local module-documented = ;
- local comment-block = ;
- local syntax-block = ;
- # This is a hack because we can not get the line of a file if it happens to
- # not have a new-line termination.
- text += "}" ;
- while $(text)
- {
- comment-block = [ extract-comment text ] ;
- syntax-block = [ extract-syntax text ] ;
- if $(.option.debug)
- {
- ECHO "HELP:" comment block; '$(comment-block)' ;
- ECHO "HELP:" syntax block; '$(syntax-block)' ;
- }
- if [ scan-rule $(syntax-block) : text ] { }
- else if [ scan-variable $(syntax-block) : text ] { }
- else if [ scan-class $(syntax-block) ] { }
- else if [ MATCH .*([cC]opyright).* : $(comment-block:J=" ") ]
- {
- # mark as the copy for the module.
- set-module-copyright $(module-name) : $(comment-block) ;
- }
- else if $(action[1]) in "print-help-project" "print-help-config"
- && ! $(jamfile<$(target)>.docs)
- {
- # special module docs for the project jamfile.
- jamfile<$(target)>.docs = $(comment-block) ;
- }
- else if ! $(module-documented)
- {
- # document the module.
- set-module-doc $(module-name) : $(comment-block) ;
- module-documented = true ;
- }
- }
- if $(action)
- {
- $(action[1]) $(module-name) : $(action[2-]) ;
- }
-}
-
-
-# Import scan-module to global scope, so that it is available during header
-# scanning phase.
-#
-IMPORT $(__name__) : scan-module : : doc.scan-module ;
-
-
-# Read in a file using the SHELL builtin and return the individual lines as
-# would be done for header scanning.
-#
-local rule read-file (
- file # The file to read in.
-)
-{
- file = [ path.native [ path.root [ path.make $(file) ] [ path.pwd ] ] ] ;
- if ! $(.file<$(file)>.lines)
- {
- local content ;
- switch [ modules.peek : OS ]
- {
- case NT :
- content = [ SHELL "TYPE \"$(file)\"" ] ;
-
- case * :
- content = [ SHELL "cat \"$(file)\"" ] ;
- }
- local lines ;
- local nl = "
-" ;
- local << = "([^$(nl)]*)[$(nl)](.*)" ;
- local line+ = [ MATCH "$(<<)" : "$(content)" ] ;
- while $(line+)
- {
- lines += $(line+[1]) ;
- line+ = [ MATCH "$(<<)" : "$(line+[2])" ] ;
- }
- .file<$(file)>.lines = $(lines) ;
- }
- return $(.file<$(file)>.lines) ;
-}
-
-
-# Add a scan action to perform to generate the help documentation. The action
-# rule is passed the name of the module as the first argument. The second
-# argument(s) are optional and passed directly as specified here.
-#
-local rule do-scan (
- modules + # The modules to scan and perform the action on.
- : action * # The action rule, plus the secondary arguments to pass to the action rule.
-)
-{
- if $(help-output) = text
- {
- print.output $(help-output-file).txt plain ;
- ALWAYS $(help-output-file).txt ;
- DEPENDS all : $(help-output-file).txt ;
- }
- if $(help-output) = html
- {
- print.output $(help-output-file).html html ;
- ALWAYS $(help-output-file).html ;
- DEPENDS all : $(help-output-file).html ;
- }
- for local module-file in $(modules[1--2])
- {
- scan-module $(module-file) : [ read-file $(module-file) ] ;
- }
- scan-module $(modules[-1]) : [ read-file $(modules[-1]) ] : $(action) ;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/util/indirect.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/util/indirect.jam b/ext/kenlm/jam-files/boost-build/util/indirect.jam
deleted file mode 100644
index 40884da..0000000
--- a/ext/kenlm/jam-files/boost-build/util/indirect.jam
+++ /dev/null
@@ -1,117 +0,0 @@
-# Copyright 2003 Dave Abrahams
-# Copyright 2003 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-import modules ;
-import numbers ;
-
-
-# The pattern that indirect rules must match: module%rule
-.pattern = ^([^%]*)%([^%]+)$ ;
-
-
-#
-# Type checking rules.
-#
-local rule indirect-rule ( x )
-{
- if ! [ MATCH $(.pattern) : $(x) ]
- {
- return "expected a string of the form module%rule, but got \""$(x)"\" for argument" ;
- }
-}
-
-
-# Make an indirect rule which calls the given rule. If context is supplied it is
-# expected to be the module in which to invoke the rule by the 'call' rule
-# below. Otherwise, the rule will be invoked in the module of this rule's
-# caller.
-#
-rule make ( rulename bound-args * : context ? )
-{
- context ?= [ CALLER_MODULE ] ;
- context ?= "" ;
- return $(context)%$(rulename) $(bound-args) ;
-}
-
-
-# Make an indirect rule which calls the given rule. 'rulename' may be a
-# qualified rule; if so it is returned unchanged. Otherwise, if frames is not
-# supplied, the result will be invoked (by 'call', below) in the module of the
-# caller. Otherwise, frames > 1 specifies additional call frames to back up in
-# order to find the module context.
-#
-rule make-qualified ( rulename bound-args * : frames ? )
-{
- if [ MATCH $(.pattern) : $(rulename) ]
- {
- return $(rulename) $(bound-args) ;
- }
- else
- {
- frames ?= 1 ;
- # If the rule name includes a Jamfile module, grab it.
- local module-context = [ MATCH ^(Jamfile<[^>]*>)\\..* : $(rulename) ] ;
-
- if ! $(module-context)
- {
- # Take the first dot-separated element as module name. This disallows
- # module names with dots, but allows rule names with dots.
- module-context = [ MATCH ^([^.]*)\\..* : $(rulename) ] ;
- }
- module-context ?= [ CALLER_MODULE $(frames) ] ;
- return [ make $(rulename) $(bound-args) : $(module-context) ] ;
- }
-}
-
-
-# Returns the module name in which the given indirect rule will be invoked.
-#
-rule get-module ( [indirect-rule] x )
-{
- local m = [ MATCH $(.pattern) : $(x) ] ;
- if ! $(m[1])
- {
- m = ;
- }
- return $(m[1]) ;
-}
-
-
-# Returns the rulename that will be called when x is invoked.
-#
-rule get-rule ( [indirect-rule] x )
-{
- local m = [ MATCH $(.pattern) : $(x) ] ;
- return $(m[2]) ;
-}
-
-
-# Invoke the given indirect-rule.
-#
-rule call ( [indirect-rule] r args * : * )
-{
- return [ modules.call-in [ get-module $(r) ] : [ get-rule $(r) ] $(args) :
- $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) : $(10) : $(11) :
- $(12) : $(13) : $(14) : $(15) : $(16) : $(17) : $(18) : $(19) ] ;
-}
-
-
-rule __test__
-{
- import assert ;
-
- rule foo-barr! ( x )
- {
- assert.equal $(x) : x ;
- }
-
- assert.equal [ get-rule [ make foo-barr! ] ] : foo-barr! ;
- assert.equal [ get-module [ make foo-barr! ] ] : [ CALLER_MODULE ] ;
-
- call [ make foo-barr! ] x ;
- call [ make foo-barr! x ] ;
- call [ make foo-barr! : [ CALLER_MODULE ] ] x ;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/util/numbers.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/util/numbers.jam b/ext/kenlm/jam-files/boost-build/util/numbers.jam
deleted file mode 100644
index 665347d..0000000
--- a/ext/kenlm/jam-files/boost-build/util/numbers.jam
+++ /dev/null
@@ -1,218 +0,0 @@
-# Copyright 2001, 2002 Dave Abrahams
-# Copyright 2002, 2003 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-import errors ;
-
-
-rule trim-leading-zeroes ( value )
-{
- return [ CALC $(value) + 0 ] ;
-}
-
-
-rule check ( numbers * )
-{
- for local n in $(numbers)
- {
- switch $(n)
- {
- case *[^0-9]* :
- errors.error $(n) "in" $(numbers) : is not a number ;
- }
- }
-}
-
-
-rule increment ( number )
-{
- return [ CALC $(number) + 1 ] ;
-}
-
-
-rule decrement ( number )
-{
- return [ CALC $(number) - 1 ] ;
-}
-
-
-rule range ( start finish ? : step ? )
-{
- if ! $(finish)
- {
- finish = $(start) ;
- start = 1 ;
- }
- step ?= 1 ;
-
- check $(start) $(finish) $(step) ;
-
- if $(finish) != 0
- {
- local result ;
- while [ less $(start) $(finish) ] || $(start) = $(finish)
- {
- result += $(start) ;
- start = [ CALC $(start) + $(step) ] ;
- }
- return $(result) ;
- }
-}
-
-
-rule less ( n1 n2 )
-{
- switch [ CALC $(n2) - $(n1) ]
- {
- case [1-9]* : return true ;
- }
-}
-
-
-rule log10 ( number )
-{
- switch $(number)
- {
- case *[^0-9]* : errors.error $(number) is not a number ;
- case 0 : errors.error can't take log of zero ;
- case [1-9] : return 0 ;
- case [1-9]? : return 1 ;
- case [1-9]?? : return 2 ;
- case [1-9]??? : return 3 ;
- case [1-9]???? : return 4 ;
- case [1-9]????? : return 5 ;
- case [1-9]?????? : return 6 ;
- case [1-9]??????? : return 7 ;
- case [1-9]???????? : return 8 ;
- case [1-9]????????? : return 9 ;
- case * :
- {
- import sequence ;
- import string ;
- local chars = [ string.chars $(number) ] ;
- while $(chars[1]) = 0
- {
- chars = $(chars[2-]) ;
- }
- if ! $(chars)
- {
- errors.error can't take log of zero ;
- }
- else
- {
- return [ decrement [ sequence.length $(chars) ] ] ;
- }
- }
- }
-}
-
-
-rule __test__ ( )
-{
- import assert ;
-
- assert.result 1 : increment 0 ;
- assert.result 2 : increment 1 ;
- assert.result 1 : decrement 2 ;
- assert.result 0 : decrement 1 ;
- assert.result 50 : increment 49 ;
- assert.result 49 : decrement 50 ;
- assert.result 99 : increment 98 ;
- assert.result 99 : decrement 100 ;
- assert.result 100 : increment 99 ;
- assert.result 999 : decrement 1000 ;
- assert.result 1000 : increment 999 ;
-
- assert.result 1 2 3 : range 3 ;
- assert.result 1 2 3 4 5 6 7 8 9 10 11 12 : range 12 ;
- assert.result 3 4 5 6 7 8 9 10 11 : range 3 11 ;
- assert.result : range 0 ;
- assert.result 1 4 7 10 : range 10 : 3 ;
- assert.result 2 4 6 8 10 : range 2 10 : 2 ;
- assert.result 25 50 75 100 : range 25 100 : 25 ;
-
- assert.result 0 : trim-leading-zeroes 0 ;
- assert.result 1234 : trim-leading-zeroes 1234 ;
- assert.result 123456 : trim-leading-zeroes 0000123456 ;
- assert.result 1000123456 : trim-leading-zeroes 1000123456 ;
- assert.result 10000 : trim-leading-zeroes 10000 ;
- assert.result 10000 : trim-leading-zeroes 00010000 ;
-
- assert.true less 1 2 ;
- assert.true less 1 12 ;
- assert.true less 1 21 ;
- assert.true less 005 217 ;
- assert.false less 0 0 ;
- assert.false less 03 3 ;
- assert.false less 3 03 ;
- assert.true less 005 217 ;
- assert.true less 0005 217 ;
- assert.true less 5 00217 ;
-
- # TEMPORARY disabled, because nested "try"/"catch" do not work and I do no
- # have the time to fix that right now.
- if $(0)
- {
- try ;
- {
- decrement 0 ;
- }
- catch can't decrement zero! ;
-
- try ;
- {
- check foo ;
- }
- catch : not a number ;
-
- try ;
- {
- increment foo ;
- }
- catch : not a number ;
-
- try ;
- {
- log10 0 ;
- }
- catch can't take log of zero ;
-
- try ;
- {
- log10 000 ;
- }
- catch can't take log of zero ;
-
- }
-
- assert.result 0 : log10 1 ;
- assert.result 0 : log10 9 ;
- assert.result 1 : log10 10 ;
- assert.result 1 : log10 99 ;
- assert.result 2 : log10 100 ;
- assert.result 2 : log10 101 ;
- assert.result 2 : log10 125 ;
- assert.result 2 : log10 999 ;
- assert.result 3 : log10 1000 ;
- assert.result 10 : log10 12345678901 ;
-
- for local x in [ range 75 110 : 5 ]
- {
- for local y in [ range $(x) 111 : 3 ]
- {
- if $(x) != $(y)
- {
- assert.true less $(x) $(y) ;
- }
- }
- }
-
- for local x in [ range 90 110 : 2 ]
- {
- for local y in [ range 80 $(x) : 4 ]
- {
- assert.false less $(x) $(y) ;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/util/option.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/util/option.jam b/ext/kenlm/jam-files/boost-build/util/option.jam
deleted file mode 100644
index f6dc375..0000000
--- a/ext/kenlm/jam-files/boost-build/util/option.jam
+++ /dev/null
@@ -1,109 +0,0 @@
-# Copyright (c) 2005 Vladimir Prus.
-#
-# Use, modification and distribution is subject to the Boost Software
-# License Version 1.0. (See accompanying file LICENSE_1_0.txt or
-# http://www.boost.org/LICENSE_1_0.txt)
-
-import modules ;
-
-# Set a value for a named option, to be used when not overridden on the command
-# line.
-rule set ( name : value ? )
-{
- .option.$(name) = $(value) ;
-}
-
-rule get ( name : default-value ? : implied-value ? )
-{
- local m = [ MATCH --$(name)=(.*) : [ modules.peek : ARGV ] ] ;
- if $(m)
- {
- return $(m[1]) ;
- }
- else
- {
- m = [ MATCH (--$(name)) : [ modules.peek : ARGV ] ] ;
- if $(m) && $(implied-value)
- {
- return $(implied-value) ;
- }
- else if $(.option.$(name))
- {
- return $(.option.$(name)) ;
- }
- else
- {
- return $(default-value) ;
- }
- }
-}
-
-
-# Check command-line args as soon as possible. For each option try to load
-# module named after option. Is that succeeds, invoke 'process' rule in the
-# module. The rule may return "true" to indicate that the regular build process
-# should not be attempted.
-#
-# Options take the general form of: --<name>[=<value>] [<value>]
-#
-rule process ( )
-{
- local ARGV = [ modules.peek : ARGV ] ;
- local BOOST_BUILD_PATH = [ modules.peek : BOOST_BUILD_PATH ] ;
-
- local dont-build ;
- local args = $(ARGV) ;
- while $(args)
- {
- local arg = [ MATCH ^--(.*) : $(args[1]) ] ;
- while $(args[2-]) && ! $(arg)
- {
- args = $(args[2-]) ;
- arg = [ MATCH ^--(.*) : $(args[1]) ] ;
- }
- args = $(args[2-]) ;
-
- if $(arg)
- {
- local split = [ MATCH ^(([^-=]+)[^=]*)(=?)(.*)$ : $(arg) ] ;
- local full-name = $(split[1]) ;
- local prefix = $(split[2]) ;
- local values ;
-
- if $(split[3])
- {
- values = $(split[4]) ;
- }
- if $(args) && ! [ MATCH ^(--).* : $(args[1]) ]
- {
- values += $(args[1]) ;
- args = $(args[2-]) ;
- }
-
- # Jook in options subdirectories of BOOST_BUILD_PATH for modules
- # matching the full option name and then its prefix.
- local plugin-dir = options ;
- local option-files = [ GLOB $(plugin-dir:D=$(BOOST_BUILD_PATH)) :
- $(full-name).jam $(prefix).jam ] ;
-
- if $(option-files)
- {
- # Load the file into a module named for the option.
- local f = $(option-files[1]) ;
- local module-name = --$(f:D=:S=) ;
- modules.load $(module-name) : $(f:D=) : $(f:D) ;
-
- # If there is a process rule, call it with the full option name
- # and its value (if any). If there was no "=" in the option, the
- # value will be empty.
- if process in [ RULENAMES $(module-name) ]
- {
- dont-build += [ modules.call-in $(module-name) : process
- --$(full-name) : $(values) ] ;
- }
- }
- }
- }
-
- return $(dont-build) ;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/util/order.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/util/order.jam b/ext/kenlm/jam-files/boost-build/util/order.jam
deleted file mode 100644
index a74fc8c..0000000
--- a/ext/kenlm/jam-files/boost-build/util/order.jam
+++ /dev/null
@@ -1,169 +0,0 @@
-# Copyright (C) 2003 Vladimir Prus
-# Use, modification, and distribution is subject to the Boost Software
-# License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy
-# at http://www.boost.org/LICENSE_1_0.txt)
-
-# This module defines a class which allows to order arbitrary object with
-# regard to arbitrary binary relation.
-#
-# The primary use case is the gcc toolset, which is sensitive to library order:
-# if library 'a' uses symbols from library 'b', then 'a' must be present before
-# 'b' on the linker's command line.
-#
-# This requirement can be lifted for gcc with GNU ld, but for gcc with Solaris
-# LD (and for Solaris toolset as well), the order always matters.
-#
-# So, we need to store order requirements and then order libraries according to
-# them. It is not possible to use the dependency graph as order requirements.
-# What we need is a "use symbols" relationship while dependency graph provides
-# the "needs to be updated" relationship.
-#
-# For example::
-# lib a : a.cpp b;
-# lib b ;
-#
-# For static linking, library 'a' need not depend on 'b'. However, it should
-# still come before 'b' on the command line.
-
-class order
-{
- rule __init__ ( )
- {
- }
-
- # Adds the constraint that 'first' should preceede 'second'.
- rule add-pair ( first second )
- {
- .constraits += $(first)--$(second) ;
- }
- NATIVE_RULE class@order : add-pair ;
-
- # Given a list of objects, reorder them so that the constraints specified by
- # 'add-pair' are satisfied.
- #
- # The algorithm was adopted from an awk script by Nikita Youshchenko
- # (yoush at cs dot msu dot su)
- rule order ( objects * )
- {
- # The algorithm used is the same is standard transitive closure, except
- # that we're not keeping in-degree for all vertices, but rather removing
- # edges.
- local result ;
- if $(objects)
- {
- local constraints = [ eliminate-unused-constraits $(objects) ] ;
-
- # Find some library that nobody depends upon and add it to the
- # 'result' array.
- local obj ;
- while $(objects)
- {
- local new_objects ;
- while $(objects)
- {
- obj = $(objects[1]) ;
- if [ has-no-dependents $(obj) : $(constraints) ]
- {
- # Emulate break ;
- new_objects += $(objects[2-]) ;
- objects = ;
- }
- else
- {
- new_objects += $(obj) ;
- obj = ;
- objects = $(objects[2-]) ;
- }
- }
-
- if ! $(obj)
- {
- errors.error "Circular order dependencies" ;
- }
- # No problem with placing first.
- result += $(obj) ;
- # Remove all contraints where 'obj' comes first, since they are
- # already satisfied.
- constraints = [ remove-satisfied $(constraints) : $(obj) ] ;
-
- # Add the remaining objects for further processing on the next
- # iteration
- objects = $(new_objects) ;
- }
-
- }
- return $(result) ;
- }
- NATIVE_RULE class@order : order ;
-
- # Eliminate constraints which mention objects not in 'objects'. In
- # graph-theory terms, this is finding a subgraph induced by ordered
- # vertices.
- rule eliminate-unused-constraits ( objects * )
- {
- local result ;
- for local c in $(.constraints)
- {
- local m = [ MATCH (.*)--(.*) : $(c) ] ;
- if $(m[1]) in $(objects) && $(m[2]) in $(objects)
- {
- result += $(c) ;
- }
- }
- return $(result) ;
- }
-
- # Returns true if there's no constraint in 'constaraints' where 'obj' comes
- # second.
- rule has-no-dependents ( obj : constraints * )
- {
- local failed ;
- while $(constraints) && ! $(failed)
- {
- local c = $(constraints[1]) ;
- local m = [ MATCH (.*)--(.*) : $(c) ] ;
- if $(m[2]) = $(obj)
- {
- failed = true ;
- }
- constraints = $(constraints[2-]) ;
- }
- if ! $(failed)
- {
- return true ;
- }
- }
-
- rule remove-satisfied ( constraints * : obj )
- {
- local result ;
- for local c in $(constraints)
- {
- local m = [ MATCH (.*)--(.*) : $(c) ] ;
- if $(m[1]) != $(obj)
- {
- result += $(c) ;
- }
- }
- return $(result) ;
- }
-}
-
-
-rule __test__ ( )
-{
- import "class" : new ;
- import assert ;
-
- c1 = [ new order ] ;
- $(c1).add-pair l1 l2 ;
-
- assert.result l1 l2 : $(c1).order l1 l2 ;
- assert.result l1 l2 : $(c1).order l2 l1 ;
-
- $(c1).add-pair l2 l3 ;
- assert.result l1 l2 : $(c1).order l2 l1 ;
- $(c1).add-pair x l2 ;
- assert.result l1 l2 : $(c1).order l2 l1 ;
- assert.result l1 l2 l3 : $(c1).order l2 l3 l1 ;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/util/os.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/util/os.jam b/ext/kenlm/jam-files/boost-build/util/os.jam
deleted file mode 100644
index daef27f..0000000
--- a/ext/kenlm/jam-files/boost-build/util/os.jam
+++ /dev/null
@@ -1,171 +0,0 @@
-# Copyright 2001, 2002, 2003, 2005 Dave Abrahams
-# Copyright 2006 Rene Rivera
-# Copyright 2003, 2005 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-
-import modules ;
-import string ;
-
-
-# Return the value(s) of the given environment variable(s) at the time bjam was
-# invoked.
-rule environ ( variable-names + )
-{
- return [ modules.peek .ENVIRON : $(variable-names) ] ;
-}
-
-.name = [ modules.peek : OS ] ;
-.platform = [ modules.peek : OSPLAT ] ;
-.version = [ modules.peek : OSVER ] ;
-
-
-local rule constant ( c : os ? )
-{
- os ?= $(.name) ;
- # First look for a platform-specific name, then the general value.
- local variables = .$(c)-$(os) .$(c) ;
- local result = $($(variables)) ;
- return $(result[1]) ;
-}
-
-rule get-constant ( os ? )
-{
- # Find the name of the constant being accessed, which is equal to the name
- # used to invoke us.
- local bt = [ BACKTRACE 1 ] ;
- local rulename = [ MATCH ([^.]*)$ : $(bt[4]) ] ;
- return [ constant $(rulename) : $(os) ] ;
-}
-
-
-# export all the common constants
-.constants = name platform version shared-library-path-variable path-separator executable-path-variable executable-suffix ;
-for local constant in $(.constants)
-{
- IMPORT $(__name__) : get-constant : $(__name__) : $(constant) ;
-}
-EXPORT $(__name__) : $(.constants) ;
-
-.executable-path-variable-NT = PATH ;
-# On Windows the case and capitalization of PATH is not always predictable, so
-# let's find out what variable name was really set.
-if $(.name) = NT
-{
- for local n in [ VARNAMES .ENVIRON ]
- {
- if $(n:L) = path
- {
- .executable-path-variable-NT = $(n) ;
- }
- }
-}
-
-# Specific constants for various platforms. There's no need to define any
-# constant whose value would be the same as the default, below.
-.shared-library-path-variable-NT = $(.executable-path-variable-NT) ;
-.path-separator-NT = ";" ;
-.expand-variable-prefix-NT = % ;
-.expand-variable-suffix-NT = % ;
-.executable-suffix-NT = .exe ;
-
-.shared-library-path-variable-CYGWIN = PATH ;
-
-.shared-library-path-variable-MACOSX = DYLD_LIBRARY_PATH ;
-
-.shared-library-path-variable-AIX = LIBPATH ;
-
-# Default constants
-.shared-library-path-variable = LD_LIBRARY_PATH ;
-.path-separator = ":" ;
-.expand-variable-prefix = $ ;
-.expand-variable-suffix = "" ;
-.executable-path-variable = PATH ;
-.executable-suffix = "" ;
-
-
-# Return a list of the directories in the PATH. Yes, that information is (sort
-# of) available in the global module, but jam code can change those values, and
-# it isn't always clear what case/capitalization to use when looking. This rule
-# is a more reliable way to get there.
-rule executable-path ( )
-{
- return [ string.words [ environ [ constant executable-path-variable ] ]
- : [ constant path-separator ] ] ;
-}
-
-
-# Initialize the list of home directories for the current user depending on the
-# OS.
-if $(.name) = NT
-{
- local home = [ environ HOMEDRIVE HOMEPATH ] ;
- .home-directories = $(home[1])$(home[2]) [ environ HOME ] [ environ USERPROFILE ] ;
-}
-else
-{
- .home-directories = [ environ HOME ] ;
-}
-
-
-# Can't use 'constant' mechanism because it only returns 1-element values.
-rule home-directories ( )
-{
- return $(.home-directories) ;
-}
-
-
-# Return the string needed to represent the expansion of the named shell
-# variable.
-rule expand-variable ( variable )
-{
- local prefix = [ constant expand-variable-prefix ] ;
- local suffix = [ constant expand-variable-suffix ] ;
- return $(prefix)$(variable)$(suffix) ;
-}
-
-
-# Returns true if running on windows, whether in cygwin or not.
-rule on-windows ( )
-{
- local result ;
- if [ modules.peek : NT ]
- {
- result = true ;
- }
- else if [ modules.peek : UNIX ]
- {
- switch [ modules.peek : JAMUNAME ]
- {
- case CYGWIN* :
- {
- result = true ;
- }
- }
- }
- return $(result) ;
-}
-
-
-if ! [ on-windows ]
-{
- .on-unix = 1 ;
-}
-
-
-rule on-unix
-{
- return $(.on-unix) ;
-}
-
-
-rule __test__
-{
- import assert ;
- if ! ( --quiet in [ modules.peek : ARGV ] )
- {
- ECHO os: name= [ name ] ;
- ECHO os: version= [ version ] ;
- }
- assert.true name ;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/util/path.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/util/path.jam b/ext/kenlm/jam-files/boost-build/util/path.jam
deleted file mode 100644
index 545d83c..0000000
--- a/ext/kenlm/jam-files/boost-build/util/path.jam
+++ /dev/null
@@ -1,910 +0,0 @@
-# Copyright 2002-2006. Vladimir Prus
-# Copyright 2003-2004. Dave Abrahams
-# Copyright 2003-2006. Rene Rivera
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# Performs various path manipulations. Paths are always in a 'normalized'
-# representation. In it, a path may be either:
-#
-# - '.', or
-#
-# - ['/'] [ ( '..' '/' )* (token '/')* token ]
-#
-# In plain english, path can be rooted, '..' elements are allowed only at the
-# beginning, and it never ends in slash, except for path consisting of slash
-# only.
-
-import modules ;
-import regex ;
-import sequence ;
-import set ;
-
-
-os = [ modules.peek : OS ] ;
-if [ modules.peek : UNIX ]
-{
- local uname = [ modules.peek : JAMUNAME ] ;
- switch $(uname)
- {
- case CYGWIN* : os = CYGWIN ;
- case * : os = UNIX ;
- }
-}
-
-
-# Converts the native path into normalized form.
-#
-rule make ( native )
-{
- return [ make-$(os) $(native) ] ;
-}
-
-
-# Builds native representation of the path.
-#
-rule native ( path )
-{
- return [ native-$(os) $(path) ] ;
-}
-
-
-# Tests if a path is rooted.
-#
-rule is-rooted ( path )
-{
- return [ MATCH "^(/)" : $(path) ] ;
-}
-
-
-# Tests if a path has a parent.
-#
-rule has-parent ( path )
-{
- if $(path) != /
- {
- return 1 ;
- }
- else
- {
- return ;
- }
-}
-
-
-# Returns the path without any directory components.
-#
-rule basename ( path )
-{
- return [ MATCH "([^/]+)$" : $(path) ] ;
-}
-
-
-# Returns parent directory of the path. If no parent exists, error is issued.
-#
-rule parent ( path )
-{
- if [ has-parent $(path) ]
- {
- if $(path) = .
- {
- return .. ;
- }
- else
- {
- # Strip everything at the end of path up to and including the last
- # slash.
- local result = [ regex.match "((.*)/)?([^/]+)" : $(path) : 2 3 ] ;
-
- # Did we strip what we shouldn't?
- if $(result[2]) = ".."
- {
- return $(path)/.. ;
- }
- else
- {
- if ! $(result[1])
- {
- if [ is-rooted $(path) ]
- {
- result = / ;
- }
- else
- {
- result = . ;
- }
- }
- return $(result[1]) ;
- }
- }
- }
- else
- {
- import errors ;
- errors.error "Path '$(path)' has no parent" ;
- }
-}
-
-
-# Returns path2 such that "[ join path path2 ] = .". The path may not contain
-# ".." element or be rooted.
-#
-rule reverse ( path )
-{
- if $(path) = .
- {
- return $(path) ;
- }
- else
- {
- local tokens = [ regex.split $(path) / ] ;
- local tokens2 ;
- for local i in $(tokens)
- {
- tokens2 += .. ;
- }
- return [ sequence.join $(tokens2) : / ] ;
- }
-}
-
-
-# Concatenates the passed path elements. Generates an error if any element other
-# than the first one is rooted. Skips any empty or undefined path elements.
-#
-rule join ( elements + )
-{
- if ! $(elements[2-])
- {
- return $(elements[1]) ;
- }
- else
- {
- for local e in $(elements[2-])
- {
- if [ is-rooted $(e) ]
- {
- import errors ;
- errors.error only the first element may be rooted ;
- }
- }
- return [ NORMALIZE_PATH "$(elements)" ] ;
- }
-}
-
-
-# If 'path' is relative, it is rooted at 'root'. Otherwise, it is unchanged.
-#
-rule root ( path root )
-{
- if [ is-rooted $(path) ]
- {
- return $(path) ;
- }
- else
- {
- return [ join $(root) $(path) ] ;
- }
-}
-
-
-# Returns the current working directory.
-#
-rule pwd ( )
-{
- if ! $(.pwd)
- {
- .pwd = [ make [ PWD ] ] ;
- }
- return $(.pwd) ;
-}
-
-
-# Returns the list of files matching the given pattern in the specified
-# directory. Both directories and patterns are supplied as portable paths. Each
-# pattern should be non-absolute path, and can't contain "." or ".." elements.
-# Each slash separated element of pattern can contain the following special
-# characters:
-# - '?', which match any character
-# - '*', which matches arbitrary number of characters.
-# A file $(d)/e1/e2/e3 (where 'd' is in $(dirs)) matches pattern p1/p2/p3 if and
-# only if e1 matches p1, e2 matches p2 and so on.
-#
-# For example:
-# [ glob . : *.cpp ]
-# [ glob . : */build/Jamfile ]
-#
-rule glob ( dirs * : patterns + : exclude-patterns * )
-{
- local result ;
- local real-patterns ;
- local real-exclude-patterns ;
- for local d in $(dirs)
- {
- for local p in $(patterns)
- {
- local pattern = [ path.root $(p) $(d) ] ;
- real-patterns += [ path.native $(pattern) ] ;
- }
-
- for local p in $(exclude-patterns)
- {
- local pattern = [ path.root $(p) $(d) ] ;
- real-exclude-patterns += [ path.native $(pattern) ] ;
- }
- }
-
- local inc = [ GLOB-RECURSIVELY $(real-patterns) ] ;
- inc = [ sequence.transform NORMALIZE_PATH : $(inc) ] ;
- local exc = [ GLOB-RECURSIVELY $(real-exclude-patterns) ] ;
- exc = [ sequence.transform NORMALIZE_PATH : $(exc) ] ;
-
- return [ sequence.transform path.make : [ set.difference $(inc) : $(exc) ] ]
- ;
-}
-
-
-# Recursive version of GLOB. Builds the glob of files while also searching in
-# the subdirectories of the given roots. An optional set of exclusion patterns
-# will filter out the matching entries from the result. The exclusions also
-# apply to the subdirectory scanning, such that directories that match the
-# exclusion patterns will not be searched.
-#
-rule glob-tree ( roots * : patterns + : exclude-patterns * )
-{
- return [ sequence.transform path.make : [ .glob-tree [ sequence.transform
- path.native : $(roots) ] : $(patterns) : $(exclude-patterns) ] ] ;
-}
-
-
-local rule .glob-tree ( roots * : patterns * : exclude-patterns * )
-{
- local excluded ;
- if $(exclude-patterns)
- {
- excluded = [ GLOB $(roots) : $(exclude-patterns) ] ;
- }
- local result = [ set.difference [ GLOB $(roots) : $(patterns) ] :
- $(excluded) ] ;
- local subdirs ;
- for local d in [ set.difference [ GLOB $(roots) : * ] : $(excluded) ]
- {
- if ! ( $(d:D=) in . .. ) && ! [ CHECK_IF_FILE $(d) ]
- {
- subdirs += $(d) ;
- }
- }
- if $(subdirs)
- {
- result += [ .glob-tree $(subdirs) : $(patterns) : $(exclude-patterns) ]
- ;
- }
- return $(result) ;
-}
-
-
-# Returns true is the specified file exists.
-#
-rule exists ( file )
-{
- return [ path.glob $(file:D) : $(file:D=) ] ;
-}
-NATIVE_RULE path : exists ;
-
-
-# Find out the absolute name of path and returns the list of all the parents,
-# starting with the immediate one. Parents are returned as relative names. If
-# 'upper_limit' is specified, directories above it will be pruned.
-#
-rule all-parents ( path : upper_limit ? : cwd ? )
-{
- cwd ?= [ pwd ] ;
- local path_ele = [ regex.split [ root $(path) $(cwd) ] / ] ;
-
- if ! $(upper_limit)
- {
- upper_limit = / ;
- }
- local upper_ele = [ regex.split [ root $(upper_limit) $(cwd) ] / ] ;
-
- # Leave only elements in 'path_ele' below 'upper_ele'.
- while $(path_ele) && ( $(upper_ele[1]) = $(path_ele[1]) )
- {
- upper_ele = $(upper_ele[2-]) ;
- path_ele = $(path_ele[2-]) ;
- }
-
- # Have all upper elements been removed ?
- if $(upper_ele)
- {
- import errors ;
- errors.error "$(upper_limit) is not prefix of $(path)" ;
- }
-
- # Create the relative paths to parents, number of elements in 'path_ele'.
- local result ;
- for local i in $(path_ele)
- {
- path = [ parent $(path) ] ;
- result += $(path) ;
- }
- return $(result) ;
-}
-
-
-# Search for 'pattern' in parent directories of 'dir', up to and including
-# 'upper_limit', if it is specified, or up to the filesystem root otherwise.
-#
-rule glob-in-parents ( dir : patterns + : upper-limit ? )
-{
- local result ;
- local parent-dirs = [ all-parents $(dir) : $(upper-limit) ] ;
-
- while $(parent-dirs) && ! $(result)
- {
- result = [ glob $(parent-dirs[1]) : $(patterns) ] ;
- parent-dirs = $(parent-dirs[2-]) ;
- }
- return $(result) ;
-}
-
-
-# Assuming 'child' is a subdirectory of 'parent', return the relative path from
-# 'parent' to 'child'.
-#
-rule relative ( child parent : no-error ? )
-{
- local not-a-child ;
- if $(parent) = "."
- {
- return $(child) ;
- }
- else
- {
- local split1 = [ regex.split $(parent) / ] ;
- local split2 = [ regex.split $(child) / ] ;
-
- while $(split1)
- {
- if $(split1[1]) = $(split2[1])
- {
- split1 = $(split1[2-]) ;
- split2 = $(split2[2-]) ;
- }
- else
- {
- not-a-child = true ;
- split1 = ;
- }
- }
- if $(split2)
- {
- if $(not-a-child)
- {
- if $(no-error)
- {
- return not-a-child ;
- }
- else
- {
- import errors ;
- errors.error $(child) is not a subdir of $(parent) ;
- }
- }
- else
- {
- return [ join $(split2) ] ;
- }
- }
- else
- {
- return "." ;
- }
- }
-}
-
-
-# Returns the minimal path to path2 that is relative to path1.
-#
-rule relative-to ( path1 path2 )
-{
- local root_1 = [ regex.split [ reverse $(path1) ] / ] ;
- local split1 = [ regex.split $(path1) / ] ;
- local split2 = [ regex.split $(path2) / ] ;
-
- while $(split1) && $(root_1)
- {
- if $(split1[1]) = $(split2[1])
- {
- root_1 = $(root_1[2-]) ;
- split1 = $(split1[2-]) ;
- split2 = $(split2[2-]) ;
- }
- else
- {
- split1 = ;
- }
- }
- return [ join . $(root_1) $(split2) ] ;
-}
-
-
-# Returns the list of paths used by the operating system for looking up
-# programs.
-#
-rule programs-path ( )
-{
- local result ;
- local raw = [ modules.peek : PATH Path path ] ;
- for local p in $(raw)
- {
- if $(p)
- {
- result += [ path.make $(p) ] ;
- }
- }
- return $(result) ;
-}
-
-
-rule makedirs ( path )
-{
- local result = true ;
- local native = [ native $(path) ] ;
- if ! [ exists $(native) ]
- {
- if [ makedirs [ parent $(path) ] ]
- {
- if ! [ MAKEDIR $(native) ]
- {
- import errors ;
- errors.error "Could not create directory '$(path)'" ;
- result = ;
- }
- }
- }
- return $(result) ;
-}
-
-
-# Converts native Windows paths into our internal canonic path representation.
-# Supports 'invalid' paths containing multiple successive path separator
-# characters.
-#
-# TODO: Check and if needed add support for Windows 'X:file' path format where
-# the file is located in the current folder on drive X.
-#
-rule make-NT ( native )
-{
- local result = [ NORMALIZE_PATH $(native) ] ;
-
- # We need to add an extra '/' in front in case this is a rooted Windows path
- # starting with a drive letter and not a path separator character since the
- # builtin NORMALIZE_PATH rule has no knowledge of this leading drive letter
- # and treats it as a regular folder name.
- if [ regex.match "(^.:)" : $(native) ]
- {
- result = /$(result) ;
- }
-
- return $(result) ;
-}
-
-
-rule native-NT ( path )
-{
- local remove-slash = [ MATCH "^/(.:.*)" : $(path) ] ;
- if $(remove-slash)
- {
- path = $(remove-slash) ;
- }
- return [ regex.replace $(path) / \\ ] ;
-}
-
-
-rule make-UNIX ( native )
-{
- # VP: I have no idea now 'native' can be empty here! But it can!
- if ! $(native)
- {
- import errors ;
- errors.error "Empty path passed to 'make-UNIX'" ;
- }
- else
- {
- return [ NORMALIZE_PATH $(native:T) ] ;
- }
-}
-
-
-rule native-UNIX ( path )
-{
- return $(path) ;
-}
-
-
-rule make-CYGWIN ( path )
-{
- return [ make-NT $(path) ] ;
-}
-
-
-rule native-CYGWIN ( path )
-{
- local result = $(path) ;
- if [ regex.match "(^/.:)" : $(path) ] # Windows absolute path.
- {
- result = [ MATCH "^/?(.*)" : $(path) ] ; # Remove leading '/'.
- }
- return [ native-UNIX $(result) ] ;
-}
-
-
-# split-path-VMS: splits input native path into device dir file (each part is
-# optional).
-#
-# example:
-#
-# dev:[dir]file.c => dev: [dir] file.c
-#
-rule split-path-VMS ( native )
-{
- local matches = [ MATCH ([a-zA-Z0-9_-]+:)?(\\[[^\]]*\\])?(.*)?$ : $(native)
- ] ;
- local device = $(matches[1]) ;
- local dir = $(matches[2]) ;
- local file = $(matches[3]) ;
-
- return $(device) $(dir) $(file) ;
-}
-
-
-# Converts a native VMS path into a portable path spec.
-#
-# Does not handle current-device absolute paths such as "[dir]File.c" as it is
-# not clear how to represent them in the portable path notation.
-#
-# Adds a trailing dot (".") to the file part if no extension is present (helps
-# when converting it back into native path).
-#
-rule make-VMS ( native )
-{
- if [ MATCH ^(\\[[a-zA-Z0-9]) : $(native) ]
- {
- import errors ;
- errors.error "Can't handle default-device absolute paths: " $(native) ;
- }
-
- local parts = [ split-path-VMS $(native) ] ;
- local device = $(parts[1]) ;
- local dir = $(parts[2]) ;
- local file = $(parts[3]) ;
- local elems ;
-
- if $(device)
- {
- #
- # rooted
- #
- elems = /$(device) ;
- }
-
- if $(dir) = "[]"
- {
- #
- # Special case: current directory
- #
- elems = $(elems) "." ;
- }
- else if $(dir)
- {
- dir = [ regex.replace $(dir) "\\[|\\]" "" ] ;
- local dir_parts = [ regex.split $(dir) \\. ] ;
-
- if $(dir_parts[1]) = ""
- {
- #
- # Relative path
- #
- dir_parts = $(dir_parts[2--1]) ;
- }
-
- #
- # replace "parent-directory" parts (- => ..)
- #
- dir_parts = [ regex.replace-list $(dir_parts) : - : .. ] ;
-
- elems = $(elems) $(dir_parts) ;
- }
-
- if $(file)
- {
- if ! [ MATCH (\\.) : $(file) ]
- {
- #
- # Always add "." to end of non-extension file.
- #
- file = $(file). ;
- }
- elems = $(elems) $(file) ;
- }
-
- local portable = [ path.join $(elems) ] ;
-
- return $(portable) ;
-}
-
-
-# Converts a portable path spec into a native VMS path.
-#
-# Relies on having at least one dot (".") included in the file name to be able
-# to differentiate it from the directory part.
-#
-rule native-VMS ( path )
-{
- local device = "" ;
- local dir = $(path) ;
- local file = "" ;
- local native ;
- local split ;
-
- #
- # Has device ?
- #
- if [ is-rooted $(dir) ]
- {
- split = [ MATCH ^/([^:]+:)/?(.*) : $(dir) ] ;
- device = $(split[1]) ;
- dir = $(split[2]) ;
- }
-
- #
- # Has file ?
- #
- # This is no exact science, just guess work:
- #
- # If the last part of the current path spec includes some chars, followed by
- # a dot, optionally followed by more chars - then it is a file (keep your
- # fingers crossed).
- #
- split = [ regex.split $(dir) / ] ;
- local maybe_file = $(split[-1]) ;
-
- if [ MATCH ^([^.]+\\..*) : $(maybe_file) ]
- {
- file = $(maybe_file) ;
- dir = [ sequence.join $(split[1--2]) : / ] ;
- }
-
- #
- # Has dir spec ?
- #
- if $(dir) = "."
- {
- dir = "[]" ;
- }
- else if $(dir)
- {
- dir = [ regex.replace $(dir) \\.\\. - ] ;
- dir = [ regex.replace $(dir) / . ] ;
-
- if $(device) = ""
- {
- #
- # Relative directory
- #
- dir = "."$(dir) ;
- }
- dir = "["$(dir)"]" ;
- }
-
- native = [ sequence.join $(device) $(dir) $(file) ] ;
-
- return $(native) ;
-}
-
-# Remove one level of indirection
-IMPORT $(__name__) : make-$(os) native-$(os) : $(__name__) : make native ;
-EXPORT $(__name__) : make native ;
-
-rule __test__ ( )
-{
- import assert ;
- import errors : try catch ;
-
- assert.true is-rooted "/" ;
- assert.true is-rooted "/foo" ;
- assert.true is-rooted "/foo/bar" ;
- assert.result : is-rooted "." ;
- assert.result : is-rooted "foo" ;
- assert.result : is-rooted "foo/bar" ;
-
- assert.true has-parent "foo" ;
- assert.true has-parent "foo/bar" ;
- assert.true has-parent "." ;
- assert.result : has-parent "/" ;
-
- assert.result "." : basename "." ;
- assert.result ".." : basename ".." ;
- assert.result "foo" : basename "foo" ;
- assert.result "foo" : basename "bar/foo" ;
- assert.result "foo" : basename "gaz/bar/foo" ;
- assert.result "foo" : basename "/gaz/bar/foo" ;
-
- assert.result "." : parent "foo" ;
- assert.result "/" : parent "/foo" ;
- assert.result "foo/bar" : parent "foo/bar/giz" ;
- assert.result ".." : parent "." ;
- assert.result ".." : parent "../foo" ;
- assert.result "../../foo" : parent "../../foo/bar" ;
-
- assert.result "." : reverse "." ;
- assert.result ".." : reverse "foo" ;
- assert.result "../../.." : reverse "foo/bar/giz" ;
-
- assert.result "foo" : join "foo" ;
- assert.result "/foo" : join "/" "foo" ;
- assert.result "foo/bar" : join "foo" "bar" ;
- assert.result "foo/bar" : join "foo/giz" "../bar" ;
- assert.result "foo/giz" : join "foo/bar/baz" "../../giz" ;
- assert.result ".." : join "." ".." ;
- assert.result ".." : join "foo" "../.." ;
- assert.result "../.." : join "../foo" "../.." ;
- assert.result "/foo" : join "/bar" "../foo" ;
- assert.result "foo/giz" : join "foo/giz" "." ;
- assert.result "." : join lib2 ".." ;
- assert.result "/" : join "/a" ".." ;
-
- assert.result /a/b : join /a/b/c .. ;
-
- assert.result "foo/bar/giz" : join "foo" "bar" "giz" ;
- assert.result "giz" : join "foo" ".." "giz" ;
- assert.result "foo/giz" : join "foo" "." "giz" ;
-
- try ;
- {
- join "a" "/b" ;
- }
- catch only first element may be rooted ;
-
- local CWD = "/home/ghost/build" ;
- assert.result : all-parents . : . : $(CWD) ;
- assert.result . .. ../.. ../../.. : all-parents "Jamfile" : "" : $(CWD) ;
- assert.result foo . .. ../.. ../../.. : all-parents "foo/Jamfile" : "" :
- $(CWD) ;
- assert.result ../Work .. ../.. ../../.. : all-parents "../Work/Jamfile" : ""
- : $(CWD) ;
-
- local CWD = "/home/ghost" ;
- assert.result . .. : all-parents "Jamfile" : "/home" : $(CWD) ;
- assert.result . : all-parents "Jamfile" : "/home/ghost" : $(CWD) ;
-
- assert.result "c/d" : relative "a/b/c/d" "a/b" ;
- assert.result "foo" : relative "foo" "." ;
-
- local save-os = [ modules.peek path : os ] ;
- modules.poke path : os : NT ;
-
- assert.result "foo/bar/giz" : make-NT "foo/bar/giz" ;
- assert.result "foo/bar/giz" : make-NT "foo\\bar\\giz" ;
- assert.result "foo" : make-NT "foo/" ;
- assert.result "foo" : make-NT "foo\\" ;
- assert.result "foo" : make-NT "foo/." ;
- assert.result "foo" : make-NT "foo/bar/.." ;
- assert.result "foo" : make-NT "foo/bar/../" ;
- assert.result "foo" : make-NT "foo/bar/..\\" ;
- assert.result "foo/bar" : make-NT "foo/././././bar" ;
- assert.result "/foo" : make-NT "\\foo" ;
- assert.result "/D:/My Documents" : make-NT "D:\\My Documents" ;
- assert.result "/c:/boost/tools/build/new/project.jam" : make-NT
- "c:\\boost\\tools\\build\\test\\..\\new\\project.jam" ;
-
- # Test processing 'invalid' paths containing multiple successive path
- # separators.
- assert.result "foo" : make-NT "foo//" ;
- assert.result "foo" : make-NT "foo///" ;
- assert.result "foo" : make-NT "foo\\\\" ;
- assert.result "foo" : make-NT "foo\\\\\\" ;
- assert.result "/foo" : make-NT "//foo" ;
- assert.result "/foo" : make-NT "///foo" ;
- assert.result "/foo" : make-NT "\\\\foo" ;
- assert.result "/foo" : make-NT "\\\\\\foo" ;
- assert.result "/foo" : make-NT "\\/\\/foo" ;
- assert.result "foo/bar" : make-NT "foo//\\//\\\\bar//\\//\\\\\\//\\//\\\\" ;
- assert.result "foo" : make-NT "foo/bar//.." ;
- assert.result "foo/bar" : make-NT "foo/bar/giz//.." ;
- assert.result "foo/giz" : make-NT
- "foo//\\//\\\\bar///\\\\//\\\\////\\/..///giz\\//\\\\\\//\\//\\\\" ;
- assert.result "../../../foo" : make-NT "..///.//..///.//..////foo///" ;
-
- # Test processing 'invalid' rooted paths with too many '..' path elements
- # that would place them before the root.
- assert.result : make-NT "/.." ;
- assert.result : make-NT "/../" ;
- assert.result : make-NT "/../." ;
- assert.result : make-NT "/.././" ;
- assert.result : make-NT "/foo/../bar/giz/.././././../../." ;
- assert.result : make-NT "/foo/../bar/giz/.././././../.././" ;
- assert.result : make-NT "//foo/../bar/giz/.././././../../." ;
- assert.result : make-NT "//foo/../bar/giz/.././././../.././" ;
- assert.result : make-NT "\\\\foo/../bar/giz/.././././../../." ;
- assert.result : make-NT "\\\\foo/../bar/giz/.././././../.././" ;
- assert.result : make-NT "/..///.//..///.//..////foo///" ;
-
- assert.result "foo\\bar\\giz" : native-NT "foo/bar/giz" ;
- assert.result "foo" : native-NT "foo" ;
- assert.result "\\foo" : native-NT "/foo" ;
- assert.result "D:\\My Documents\\Work" : native-NT "/D:/My Documents/Work" ;
-
- modules.poke path : os : UNIX ;
-
- assert.result "foo/bar/giz" : make-UNIX "foo/bar/giz" ;
- assert.result "/sub1" : make-UNIX "/sub1/." ;
- assert.result "/sub1" : make-UNIX "/sub1/sub2/.." ;
- assert.result "sub1" : make-UNIX "sub1/." ;
- assert.result "sub1" : make-UNIX "sub1/sub2/.." ;
- assert.result "/foo/bar" : native-UNIX "/foo/bar" ;
-
- modules.poke path : os : VMS ;
-
- #
- # Do not really need to poke os before these
- #
- assert.result "disk:" "[dir]" "file" : split-path-VMS "disk:[dir]file" ;
- assert.result "disk:" "[dir]" "" : split-path-VMS "disk:[dir]" ;
- assert.result "disk:" "" "" : split-path-VMS "disk:" ;
- assert.result "disk:" "" "file" : split-path-VMS "disk:file" ;
- assert.result "" "[dir]" "file" : split-path-VMS "[dir]file" ;
- assert.result "" "[dir]" "" : split-path-VMS "[dir]" ;
- assert.result "" "" "file" : split-path-VMS "file" ;
- assert.result "" "" "" : split-path-VMS "" ;
-
- #
- # Special case: current directory
- #
- assert.result "" "[]" "" : split-path-VMS "[]" ;
- assert.result "disk:" "[]" "" : split-path-VMS "disk:[]" ;
- assert.result "" "[]" "file" : split-path-VMS "[]file" ;
- assert.result "disk:" "[]" "file" : split-path-VMS "disk:[]file" ;
-
- #
- # Make portable paths
- #
- assert.result "/disk:" : make-VMS "disk:" ;
- assert.result "foo/bar/giz" : make-VMS "[.foo.bar.giz]" ;
- assert.result "foo" : make-VMS "[.foo]" ;
- assert.result "foo" : make-VMS "[.foo.bar.-]" ;
- assert.result ".." : make-VMS "[.-]" ;
- assert.result ".." : make-VMS "[-]" ;
- assert.result "." : make-VMS "[]" ;
- assert.result "giz.h" : make-VMS "giz.h" ;
- assert.result "foo/bar/giz.h" : make-VMS "[.foo.bar]giz.h" ;
- assert.result "/disk:/my_docs" : make-VMS "disk:[my_docs]" ;
- assert.result "/disk:/boost/tools/build/new/project.jam" : make-VMS
- "disk:[boost.tools.build.test.-.new]project.jam" ;
-
- #
- # Special case (adds '.' to end of file w/o extension to disambiguate from
- # directory in portable path spec)
- #
- assert.result "Jamfile." : make-VMS "Jamfile" ;
- assert.result "dir/Jamfile." : make-VMS "[.dir]Jamfile" ;
- assert.result "/disk:/dir/Jamfile." : make-VMS "disk:[dir]Jamfile" ;
-
- #
- # Make native paths
- #
- assert.result "disk:" : native-VMS "/disk:" ;
- assert.result "[.foo.bar.giz]" : native-VMS "foo/bar/giz" ;
- assert.result "[.foo]" : native-VMS "foo" ;
- assert.result "[.-]" : native-VMS ".." ;
- assert.result "[.foo.-]" : native-VMS "foo/.." ;
- assert.result "[]" : native-VMS "." ;
- assert.result "disk:[my_docs.work]" : native-VMS "/disk:/my_docs/work" ;
- assert.result "giz.h" : native-VMS "giz.h" ;
- assert.result "disk:Jamfile." : native-VMS "/disk:Jamfile." ;
- assert.result "disk:[my_docs.work]Jamfile." : native-VMS
- "/disk:/my_docs/work/Jamfile." ;
-
- modules.poke path : os : $(save-os) ;
-}
[44/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/build/scanner.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/build/scanner.jam b/ext/kenlm/jam-files/boost-build/build/scanner.jam
deleted file mode 100644
index ed55070..0000000
--- a/ext/kenlm/jam-files/boost-build/build/scanner.jam
+++ /dev/null
@@ -1,163 +0,0 @@
-# Copyright 2003 Dave Abrahams
-# Copyright 2002, 2003, 2004, 2005 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# Implements scanners: objects computing implicit dependencies for files, such
-# as includes in C++.
-#
-# A scanner has a regular expression used to find the dependencies, some data
-# needed to interpret those dependencies (e.g., include paths), and code which
-# establishing needed relationships between actual jam targets.
-#
-# Scanner objects are created by actions when they try to actualize virtual
-# targets, passed to the virtual-target.actualize() method and are then
-# associated with actual targets. It is possible to use several scanners for a
-# single virtual-target. For example, a single source file might be compiled
-# twice - each time using a different include path. In this case, two separate
-# actual targets will be created, each having a scanner of its own.
-#
-# Typically, scanners are created from target type and the action's properties,
-# using the rule 'get' in this module. Directly creating scanners is not
-# recommended, as it might create multiple equvivalent but different instances,
-# and lead to unnecessary actual target duplication. However, actions can also
-# create scanners in a special way, instead of relying on just the target type.
-
-import "class" : new ;
-import property ;
-import property-set ;
-import virtual-target ;
-
-# Base scanner class.
-#
-class scanner
-{
- rule __init__ ( )
- {
- }
-
- # Returns a pattern to use for scanning.
- #
- rule pattern ( )
- {
- import errors : error : errors.error ;
- errors.error "method must be overriden" ;
- }
-
- # Establish necessary relationship between targets, given an actual target
- # beeing scanned and a list of pattern matches in that file.
- #
- rule process ( target : matches * )
- {
- import errors : error : errors.error ;
- errors.error "method must be overriden" ;
- }
-}
-
-
-# Registers a new generator class, specifying a set of properties relevant to
-# this scanner. Constructor for that class should have one parameter: a list of
-# properties.
-#
-rule register ( scanner-class : relevant-properties * )
-{
- .registered += $(scanner-class) ;
- .relevant-properties.$(scanner-class) = $(relevant-properties) ;
-}
-
-
-# Common scanner class, usable when there is only one kind of includes (unlike
-# C, where "" and <> includes have different search paths).
-#
-class common-scanner : scanner
-{
- import scanner ;
-
- rule __init__ ( includes * )
- {
- scanner.__init__ ;
- self.includes = $(includes) ;
- }
-
- rule process ( target : matches * : binding )
- {
- local target_path = [ NORMALIZE_PATH $(binding:D) ] ;
-
- NOCARE $(matches) ;
- INCLUDES $(target) : $(matches) ;
- SEARCH on $(matches) = $(target_path) $(self.includes:G=) ;
- ISFILE $(matches) ;
-
- scanner.propagate $(__name__) : $(matches) : $(target) ;
- }
-}
-
-
-# Returns an instance of a previously registered scanner, with the specified
-# properties.
-#
-rule get ( scanner-class : property-set )
-{
- if ! $(scanner-class) in $(.registered)
- {
- import errors ;
- errors.error "attempt to get an unregisted scanner" ;
- }
-
- local r = $(.rv-cache.$(property-set)) ;
- if ! $(r)
- {
- r = [ property-set.create
- [ property.select $(.relevant-properties.$(scanner-class)) :
- [ $(property-set).raw ] ] ] ;
- .rv-cache.$(property-set) = $(r) ;
- }
-
- if ! $(scanner.$(scanner-class).$(r:J=-))
- {
- local s = [ new $(scanner-class) [ $(r).raw ] ] ;
- scanner.$(scanner-class).$(r:J=-) = $(s) ;
- }
- return $(scanner.$(scanner-class).$(r:J=-)) ;
-}
-
-
-# Installs the specified scanner on the actual target 'target'.
-#
-rule install ( scanner : target )
-{
- HDRSCAN on $(target) = [ $(scanner).pattern ] ;
- SCANNER on $(target) = $(scanner) ;
- HDRRULE on $(target) = scanner.hdrrule ;
-
- # Scanner reflects differences in properties affecting binding of 'target',
- # which will be known when processing includes for it, and give information
- # on how to interpret different include types (e.g. quoted vs. those in
- # angle brackets in C files).
- HDRGRIST on $(target) = $(scanner) ;
-}
-
-
-# Propagate scanner settings from 'including-target' to 'targets'.
-#
-rule propagate ( scanner : targets * : including-target )
-{
- HDRSCAN on $(targets) = [ on $(including-target) return $(HDRSCAN) ] ;
- SCANNER on $(targets) = $(scanner) ;
- HDRRULE on $(targets) = scanner.hdrrule ;
- HDRGRIST on $(targets) = [ on $(including-target) return $(HDRGRIST) ] ;
-}
-
-
-rule hdrrule ( target : matches * : binding )
-{
- local scanner = [ on $(target) return $(SCANNER) ] ;
- $(scanner).process $(target) : $(matches) : $(binding) ;
-}
-
-
-# hdrrule must be available at global scope so it can be invoked by header
-# scanning.
-#
-IMPORT scanner : hdrrule : : scanner.hdrrule ;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/build/targets.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/build/targets.jam b/ext/kenlm/jam-files/boost-build/build/targets.jam
deleted file mode 100644
index 3bd39a3..0000000
--- a/ext/kenlm/jam-files/boost-build/build/targets.jam
+++ /dev/null
@@ -1,1694 +0,0 @@
-# Copyright Vladimir Prus 2002.
-# Copyright Rene Rivera 2006.
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# Supports 'abstract' targets, which are targets explicitly defined in a
-# Jamfile.
-#
-# Abstract targets are represented by classes derived from 'abstract-target'
-# class. The first abstract target is 'project-target', which is created for
-# each Jamfile, and can be obtained by the 'target' rule in the Jamfile's module
-# (see project.jam).
-#
-# Project targets keep a list of 'main-target' instances. A main target is what
-# the user explicitly defines in a Jamfile. It is possible to have several
-# definitions for a main target, for example to have different lists of sources
-# for different platforms. So, main targets keep a list of alternatives.
-#
-# Each alternative is an instance of 'abstract-target'. When a main target
-# subvariant is defined by some rule, that rule will decide what class to use,
-# create an instance of that class and add it to the list of alternatives for
-# the main target.
-#
-# Rules supplied by the build system will use only targets derived from the
-# 'basic-target' class, which will provide some default behaviour. There will be
-# different classes derived from it such as 'make-target', created by the 'make'
-# rule, and 'typed-target', created by rules such as 'exe' and 'lib'.
-#
-# +--------------------------+
-# | abstract-target |
-# +==========================+
-# | name |
-# | project |
-# | |
-# | generate(properties) = 0 |
-# +-------------+------------+
-# |
-# ^
-# / \
-# +-+-+
-# |
-# |
-# +------------------+-----+-------------------------------+
-# | | |
-# | | |
-# +-----------+----------+ +------+------+ +-------+------+
-# | project-target | | main-target | | basic-target |
-# +======================+ 1 * +=============+ alternatives +==============+
-# | generate(properties) |o-----+ generate |<>------------->| generate |
-# | main-target | +-------------+ | construct = 0|
-# +----------------------+ +-------+------+
-# |
-# ^
-# / \
-# +-+-+
-# |
-# |
-# ...--+-----------------+-----------------+------------------+
-# | | | |
-# | | | |
-# ... ---+-----+ +-------+------+ +------+------+ +-------+------+
-# | | typed-target | | make-target | | stage-target |
-# . +==============+ +=============+ +==============+
-# . | construct | | construct | | construct |
-# +--------------+ +-------------+ +--------------+
-
-import assert ;
-import build-request ;
-import "class" : new ;
-import feature ;
-import indirect ;
-import path ;
-import property ;
-import property-set ;
-import sequence ;
-import set ;
-import toolset ;
-
-
-# Base class for all abstract targets.
-#
-class abstract-target
-{
- import assert ;
- import "class" ;
- import errors ;
- import project ;
-
- rule __init__ ( name # Name of the target in Jamfile.
- : project-target # The project target to which this one belongs.
- )
- {
- # Note: it might seem that we don't need either name or project at all.
- # However, there are places where we really need it. One example is
- # error messages which should name problematic targets. Another is
- # setting correct paths for sources and generated files.
-
- self.name = $(name) ;
- self.project = $(project-target) ;
- self.location = [ errors.nearest-user-location ] ;
- }
-
- # Returns the name of this target.
- rule name ( )
- {
- return $(self.name) ;
- }
-
- # Returns the project for this target.
- rule project ( )
- {
- return $(self.project) ;
- }
-
- # Return the location where the target was declared.
- rule location ( )
- {
- return $(self.location) ;
- }
-
- # Returns a user-readable name for this target.
- rule full-name ( )
- {
- local location = [ $(self.project).get location ] ;
- return $(location)/$(self.name) ;
- }
-
- # Generates virtual targets for this abstract target using the specified
- # properties, unless a different value of some feature is required by the
- # target.
- # On success, returns:
- # - a property-set with the usage requirements to be applied to dependants
- # - a list of produced virtual targets, which may be empty.
- # If 'property-set' is empty, performs the default build of this target, in
- # a way specific to the derived class.
- #
- rule generate ( property-set )
- {
- errors.error "method should be defined in derived classes" ;
- }
-
- rule rename ( new-name )
- {
- self.name = $(new-name) ;
- }
-}
-
-
-if --debug-building in [ modules.peek : ARGV ]
-{
- modules.poke : .debug-building : true ;
-}
-
-
-rule indent ( )
-{
- return $(.indent:J="") ;
-}
-
-
-rule increase-indent ( )
-{
- .indent += " " ;
-}
-
-
-rule decrease-indent ( )
-{
- .indent = $(.indent[2-]) ;
-}
-
-
-# Project target class (derived from 'abstract-target').
-#
-# This class has the following responsibilities:
-# - Maintaining a list of main targets in this project and building them.
-#
-# Main targets are constructed in two stages:
-# - When Jamfile is read, a number of calls to 'add-alternative' is made. At
-# that time, alternatives can also be renamed to account for inline targets.
-# - The first time 'main-target' or 'has-main-target' rule is called, all
-# alternatives are enumerated and main targets are created.
-#
-class project-target : abstract-target
-{
- import project ;
- import targets ;
- import path ;
- import print ;
- import property-set ;
- import set ;
- import sequence ;
- import "class" : new ;
-
- rule __init__ ( name : project-module parent-project ?
- : requirements * : default-build * )
- {
- abstract-target.__init__ $(name) : $(__name__) ;
-
- self.project-module = $(project-module) ;
- self.location = [ project.attribute $(project-module) location ] ;
- self.requirements = $(requirements) ;
- self.default-build = $(default-build) ;
-
- if $(parent-project)
- {
- inherit $(parent-project) ;
- }
- }
-
- # This is needed only by the 'make' rule. Need to find a way to make 'make'
- # work without this method.
- #
- rule project-module ( )
- {
- return $(self.project-module) ;
- }
-
- rule get ( attribute )
- {
- return [ project.attribute $(self.project-module) $(attribute) ] ;
- }
-
- rule build-dir ( )
- {
- if ! $(self.build-dir)
- {
- self.build-dir = [ get build-dir ] ;
- if ! $(self.build-dir)
- {
- local location = [ $(self.project).get location ] ;
- if $(location)
- {
- self.build-dir = [ path.join $(location) bin ] ;
- }
- else
- {
- local id = [ get id ] ;
- if $(id)
- {
- local rid = [ MATCH ^/(.*) : $(id) ] ;
- self.build-dir = [ path.join [ project.standalone-build-dir ] $(rid) ] ;
- }
- else
- {
- errors.error "Could not create build-dir for standalone project $(self.project-module:E=)."
- : "Missing project id" ;
- }
- }
- }
- }
- return $(self.build-dir) ;
- }
-
- # Generates all possible targets contained in this project.
- #
- rule generate ( property-set * )
- {
- if [ modules.peek : .debug-building ]
- {
- ECHO [ targets.indent ] "building project" [ name ]
- " ('$(__name__)') with" [ $(property-set).raw ] ;
- targets.increase-indent ;
- }
-
- local usage-requirements = [ property-set.empty ] ;
- local targets ;
-
- for local t in [ targets-to-build ]
- {
- local g = [ $(t).generate $(property-set) ] ;
- usage-requirements = [ $(usage-requirements).add $(g[1]) ] ;
- targets += $(g[2-]) ;
- }
- targets.decrease-indent ;
- return $(usage-requirements) [ sequence.unique $(targets) ] ;
- }
-
- # Computes and returns a list of abstract-target instances which must be
- # built when this project is built.
- #
- rule targets-to-build ( )
- {
- local result ;
-
- if ! $(self.built-main-targets)
- {
- build-main-targets ;
- }
-
- # Collect all main targets here, except for "explicit" ones.
- for local t in $(self.main-targets)
- {
- if ! [ $(t).name ] in $(self.explicit-targets)
- {
- result += $(t) ;
- }
- }
-
- # Collect all projects referenced via "projects-to-build" attribute.
- local self-location = [ get location ] ;
- for local pn in [ get projects-to-build ]
- {
- result += [ find $(pn)/ ] ;
- }
-
- return $(result) ;
- }
-
- # Add 'target' to the list of targets in this project that should be build
- # only by explicit request
- #
- rule mark-target-as-explicit ( target-name * )
- {
- # Record the name of the target, not instance, since this rule is called
- # before main target instances are created.
- self.explicit-targets += $(target-name) ;
- }
-
- rule mark-target-as-always ( target-name * )
- {
- # Record the name of the target, not instance, since this rule is called
- # before main target instances are created.
- self.always-targets += $(target-name) ;
- }
-
- # Add new target alternative
- #
- rule add-alternative ( target-instance )
- {
- if $(self.built-main-targets)
- {
- import errors : error : errors.error ;
- errors.error add-alternative called when main targets are already
- created. : in project [ full-name ] ;
- }
- self.alternatives += $(target-instance) ;
- }
-
- # Returns a 'main-target' class instance corresponding to 'name'.
- #
- rule main-target ( name )
- {
- if ! $(self.built-main-targets)
- {
- build-main-targets ;
- }
- return $(self.main-target.$(name)) ;
- }
-
- # Returns whether a main target with the specified name exists.
- #
- rule has-main-target ( name )
- {
- if ! $(self.built-main-targets)
- {
- build-main-targets ;
- }
-
- if $(self.main-target.$(name))
- {
- return true ;
- }
- }
-
- # Worker function for the find rule not implementing any caching and simply
- # returning nothing in case the target can not be found.
- #
- rule find-really ( id )
- {
- local result ;
- local current-location = [ get location ] ;
-
- local split = [ MATCH ^(.*)//(.*)$ : $(id) ] ;
- local project-part = $(split[1]) ;
- local target-part = $(split[2]) ;
-
- local extra-error-message ;
- if $(project-part)
- {
- # There is an explicitly specified project part in id. Looks up the
- # project and passes the request to it.
- local pm = [ project.find $(project-part) : $(current-location) ] ;
- if $(pm)
- {
- project-target = [ project.target $(pm) ] ;
- result = [ $(project-target).find $(target-part) : no-error ] ;
- }
- else
- {
- extra-error-message = could not resolve project reference
- '$(project-part)' ;
- if ! [ path.is-rooted $(project-part) ]
- {
- local rooted = [ path.root $(project-part) / ] ;
- if [ project.is-registered-id $(rooted) ]
- {
- extra-error-message += - possibly missing a leading
- slash ('/') character. ;
- }
- }
- }
- }
- else
- {
- # Interpret target-name as name of main target. Need to do this
- # before checking for file. Consider the following scenario with a
- # toolset not modifying its executable's names, e.g. gcc on
- # Unix-like platforms:
- #
- # exe test : test.cpp ;
- # install s : test : <location>. ;
- #
- # After the first build we would have a target named 'test' in the
- # Jamfile and a file named 'test' on the disk. We need the target to
- # override the file.
- result = [ main-target $(id) ] ;
-
- # Interpret id as an existing file reference.
- if ! $(result)
- {
- result = [ new file-reference [ path.make $(id) ] :
- $(self.project) ] ;
- if ! [ $(result).exists ]
- {
- result = ;
- }
- }
-
- # Interpret id as project-id.
- if ! $(result)
- {
- local project-module = [ project.find $(id) :
- $(current-location) ] ;
- if $(project-module)
- {
- result = [ project.target $(project-module) ] ;
- }
- }
- }
-
- return $(result:E="") $(extra-error-message) ;
- }
-
- # Find and return the target with the specified id, treated relative to
- # self. Id may specify either a target or a file name with the target taking
- # priority. May report an error or return nothing if the target is not found
- # depending on the 'no-error' parameter.
- #
- rule find ( id : no-error ? )
- {
- local v = $(.id.$(id)) ;
- local extra-error-message ;
- if ! $(v)
- {
- local r = [ find-really $(id) ] ;
- v = $(r[1]) ;
- extra-error-message = $(r[2-]) ;
- if ! $(v)
- {
- v = none ;
- }
- .id.$(id) = $(v) ;
- }
-
- if $(v) != none
- {
- return $(v) ;
- }
- else if ! $(no-error)
- {
- local current-location = [ get location ] ;
- import errors : user-error : errors.user-error ;
- errors.user-error Unable to find file or target named
- : " " '$(id)'
- : referred to from project at
- : " " '$(current-location)'
- : $(extra-error-message) ;
- }
- }
-
- rule build-main-targets ( )
- {
- self.built-main-targets = true ;
- for local a in $(self.alternatives)
- {
- local name = [ $(a).name ] ;
- local target = $(self.main-target.$(name)) ;
- if ! $(target)
- {
- local t = [ new main-target $(name) : $(self.project) ] ;
- self.main-target.$(name) = $(t) ;
- self.main-targets += $(t) ;
- target = $(self.main-target.$(name)) ;
- }
-
- if $(name) in $(self.always-targets)
- {
- $(a).always ;
- }
-
- $(target).add-alternative $(a) ;
- }
- }
-
- # Accessor, add a constant.
- #
- rule add-constant (
- name # Variable name of the constant.
- : value + # Value of the constant.
- : type ? # Optional type of value.
- )
- {
- switch $(type)
- {
- case path :
- local r ;
- for local v in $(value)
- {
- local l = $(self.location) ;
- if ! $(l)
- {
- # Project corresponding to config files do not have
- # 'location' attribute, but do have source location. It
- # might be more reasonable to make every project have a
- # location and use some other approach to prevent buildable
- # targets in config files, but that has been left for later.
- l = [ get source-location ] ;
- }
- v = [ path.root [ path.make $(v) ] $(l) ] ;
- # Now make the value absolute path.
- v = [ path.root $(v) [ path.pwd ] ] ;
- # Constants should be in platform-native form.
- v = [ path.native $(v) ] ;
- r += $(v) ;
- }
- value = $(r) ;
- }
- if ! $(name) in $(self.constants)
- {
- self.constants += $(name) ;
- }
- self.constant.$(name) = $(value) ;
- # Inject the constant in the scope of the Jamroot module.
- modules.poke $(self.project-module) : $(name) : $(value) ;
- }
-
- rule inherit ( parent )
- {
- for local c in [ modules.peek $(parent) : self.constants ]
- {
- # No need to pass the type. Path constants were converted to
- # absolute paths already by parent.
- add-constant $(c) : [ modules.peek $(parent) : self.constant.$(c) ]
- ;
- }
-
- # Import rules from parent.
- local this-module = [ project-module ] ;
- local parent-module = [ $(parent).project-module ] ;
- # Do not import rules coming from 'project-rules' as they must be
- # imported localized.
- local user-rules = [ set.difference
- [ RULENAMES $(parent-module) ] :
- [ RULENAMES project-rules ] ] ;
- IMPORT $(parent-module) : $(user-rules) : $(this-module) : $(user-rules)
- ;
- EXPORT $(this-module) : $(user-rules) ;
- }
-}
-
-
-# Helper rules to detect cycles in main target references.
-#
-local rule start-building ( main-target-instance )
-{
- if $(main-target-instance) in $(.targets-being-built)
- {
- local names ;
- for local t in $(.targets-being-built) $(main-target-instance)
- {
- names += [ $(t).full-name ] ;
- }
-
- import errors ;
- errors.error "Recursion in main target references"
- : "the following target are being built currently:"
- : $(names) ;
- }
- .targets-being-built += $(main-target-instance) ;
-}
-
-
-local rule end-building ( main-target-instance )
-{
- .targets-being-built = $(.targets-being-built[1--2]) ;
-}
-
-
-# A named top-level target in Jamfile.
-#
-class main-target : abstract-target
-{
- import assert ;
- import feature ;
- import print ;
- import property-set ;
- import sequence ;
- import targets : start-building end-building ;
-
- rule __init__ ( name : project )
- {
- abstract-target.__init__ $(name) : $(project) ;
- }
-
- # Add a new alternative for this target
- rule add-alternative ( target )
- {
- local d = [ $(target).default-build ] ;
- if $(self.alternatives) && ( $(self.default-build) != $(d) )
- {
- import errors : error : errors.error ;
- errors.error "default build must be identical in all alternatives"
- : "main target is" [ full-name ]
- : "with" [ $(d).raw ]
- : "differing from previous default build"
- [ $(self.default-build).raw ] ;
- }
- else
- {
- self.default-build = $(d) ;
- }
- self.alternatives += $(target) ;
- }
-
- # Returns the best viable alternative for this property-set. See the
- # documentation for selection rules.
- #
- local rule select-alternatives ( property-set debug ? )
- {
- # When selecting alternatives we have to consider defaults, for example:
- # lib l : l.cpp : <variant>debug ;
- # lib l : l_opt.cpp : <variant>release ;
- # will not work unless we add default value <variant>debug.
- property-set = [ $(p).add-defaults ] ;
-
- # The algorithm: we keep the current best viable alternative. When we
- # encounter a new best viable alternative, we compare it with the
- # current one.
-
- local best ;
- local best-properties ;
-
- if $(self.alternatives[2-])
- {
- local bad ;
- local worklist = $(self.alternatives) ;
- while $(worklist) && ! $(bad)
- {
- local v = $(worklist[1]) ;
- local properties = [ $(v).match $(property-set) $(debug) ] ;
-
- if $(properties) != no-match
- {
- if ! $(best)
- {
- best = $(v) ;
- best-properties = $(properties) ;
- }
- else
- {
- if $(properties) = $(best-properties)
- {
- bad = true ;
- }
- else if $(properties) in $(best-properties)
- {
- # Do nothing, this alternative is worse
- }
- else if $(best-properties) in $(properties)
- {
- best = $(v) ;
- best-properties = $(properties) ;
- }
- else
- {
- bad = true ;
- }
- }
- }
- worklist = $(worklist[2-]) ;
- }
- if ! $(bad)
- {
- return $(best) ;
- }
- }
- else
- {
- return $(self.alternatives) ;
- }
- }
-
- rule apply-default-build ( property-set )
- {
- return [ targets.apply-default-build $(property-set) :
- $(self.default-build) ] ;
- }
-
- # Select an alternative for this main target, by finding all alternatives
- # whose requirements are satisfied by 'properties' and picking the one with
- # the longest requirements set. Returns the result of calling 'generate' on
- # that alternative.
- #
- rule generate ( property-set )
- {
- start-building $(__name__) ;
-
- # We want composite properties in the build request to act as if all the
- # properties they expand to have been explicitly specified.
- property-set = [ $(property-set).expand ] ;
-
- local all-property-sets = [ apply-default-build $(property-set) ] ;
- local usage-requirements = [ property-set.empty ] ;
- local result ;
- for local p in $(all-property-sets)
- {
- local r = [ generate-really $(p) ] ;
- if $(r)
- {
- usage-requirements = [ $(usage-requirements).add $(r[1]) ] ;
- result += $(r[2-]) ;
- }
- }
- end-building $(__name__) ;
- return $(usage-requirements) [ sequence.unique $(result) ] ;
- }
-
- # Generates the main target with the given property set and returns a list
- # which first element is property-set object containing usage-requirements
- # of generated target and with generated virtual target in other elements.
- # It is possible that no targets are generated.
- #
- local rule generate-really ( property-set )
- {
- local best-alternatives = [ select-alternatives $(property-set) ] ;
- if ! $(best-alternatives)
- {
- ECHO "error: No best alternative for" [ full-name ] ;
- select-alternatives $(property-set) debug ;
- return [ property-set.empty ] ;
- }
- else
- {
- # Now return virtual targets for the only alternative.
- return [ $(best-alternatives).generate $(property-set) ] ;
- }
- }
-
- rule rename ( new-name )
- {
- abstract-target.rename $(new-name) ;
- for local a in $(self.alternatives)
- {
- $(a).rename $(new-name) ;
- }
- }
-}
-
-
-# Abstract target referring to a source file. This is an artificial entity
-# allowing sources to a target to be represented using a list of abstract target
-# instances.
-#
-class file-reference : abstract-target
-{
- import virtual-target ;
- import property-set ;
- import path ;
-
- rule __init__ ( file : project )
- {
- abstract-target.__init__ $(file) : $(project) ;
- }
-
- rule generate ( properties )
- {
- return [ property-set.empty ] [ virtual-target.from-file $(self.name) :
- [ location ] : $(self.project) ] ;
- }
-
- # Returns true if the referred file really exists.
- rule exists ( )
- {
- location ;
- return $(self.file-path) ;
- }
-
- # Returns the location of target. Needed by 'testing.jam'.
- rule location ( )
- {
- if ! $(self.file-location)
- {
- local source-location = [ $(self.project).get source-location ] ;
- for local src-dir in $(source-location)
- {
- if ! $(self.file-location)
- {
- local location = [ path.root $(self.name) $(src-dir) ] ;
- if [ CHECK_IF_FILE [ path.native $(location) ] ]
- {
- self.file-location = $(src-dir) ;
- self.file-path = $(location) ;
- }
- }
- }
- }
- return $(self.file-location) ;
- }
-}
-
-
-# Given a target-reference, made in context of 'project', returns the
-# abstract-target instance that is referred to, as well as properties explicitly
-# specified for this reference.
-#
-rule resolve-reference ( target-reference : project )
-{
- # Separate target name from properties override.
- local split = [ MATCH "^([^<]*)(/(<.*))?$" : $(target-reference) ] ;
- local id = $(split[1]) ;
- local sproperties = ;
- if $(split[3])
- {
- sproperties = [ property.make [ feature.split $(split[3]) ] ] ;
- sproperties = [ feature.expand-composites $(sproperties) ] ;
- }
-
- # Find the target.
- local target = [ $(project).find $(id) ] ;
-
- return $(target) [ property-set.create $(sproperties) ] ;
-}
-
-
-# Attempts to generate the target given by target reference, which can refer
-# both to a main target or to a file. Returns a list consisting of
-# - usage requirements
-# - generated virtual targets, if any
-#
-rule generate-from-reference (
- target-reference # Target reference.
- : project # Project where the reference is made.
- : property-set # Properties of the main target that makes the reference.
-)
-{
- local r = [ resolve-reference $(target-reference) : $(project) ] ;
- local target = $(r[1]) ;
- local sproperties = $(r[2]) ;
-
- # Take properties which should be propagated and refine them with
- # source-specific requirements.
- local propagated = [ $(property-set).propagated ] ;
- local rproperties = [ $(propagated).refine $(sproperties) ] ;
- if $(rproperties[1]) = "@error"
- {
- import errors ;
- errors.error
- "When building" [ full-name ] " with properties " $(properties) :
- "Invalid properties specified for " $(source) ":"
- $(rproperties[2-]) ;
- }
- return [ $(target).generate $(rproperties) ] ;
-}
-
-
-rule apply-default-build ( property-set : default-build )
-{
- # 1. First, see what properties from default-build are already present in
- # property-set.
-
- local raw = [ $(property-set).raw ] ;
- local specified-features = $(raw:G) ;
-
- local defaults-to-apply ;
- for local d in [ $(default-build).raw ]
- {
- if ! $(d:G) in $(specified-features)
- {
- defaults-to-apply += $(d) ;
- }
- }
-
- # 2. If there are any defaults to be applied, form a new build request. Pass
- # it through to 'expand-no-defaults' since default-build might contain
- # "release debug" resulting in two property-sets.
- local result ;
- if $(defaults-to-apply)
- {
- # We have to compress subproperties here to prevent property lists like:
- # <toolset>msvc <toolset-msvc:version>7.1 <threading>multi
- #
- # from being expanded into:
- # <toolset-msvc:version>7.1/<threading>multi
- # <toolset>msvc/<toolset-msvc:version>7.1/<threading>multi
- #
- # due to a cross-product property combination. That may be an indication
- # that build-request.expand-no-defaults is the wrong rule to use here.
- properties = [ build-request.expand-no-defaults
- [ feature.compress-subproperties $(raw) ] $(defaults-to-apply) ] ;
-
- if $(properties)
- {
- for local p in $(properties)
- {
- result += [ property-set.create
- [ feature.expand [ feature.split $(p) ] ] ] ;
- }
- }
- else
- {
- result = [ property-set.empty ] ;
- }
- }
- else
- {
- result = $(property-set) ;
- }
- return $(result) ;
-}
-
-
-# Given a build request and requirements, return properties common to dependency
-# build request and target requirements.
-#
-# TODO: Document exactly what 'common properties' are, whether they should
-# include default property values, whether they should contain any conditional
-# properties or should those be already processed, etc. See whether there are
-# any differences between use cases with empty and non-empty build-request as
-# well as with requirements containing and those not containing any non-free
-# features.
-#
-rule common-properties ( build-request requirements )
-{
- # For optimization, we add free requirements directly, without using a
- # complex algorithm. This gives the complex algorithm a better chance of
- # caching results.
- local free = [ $(requirements).free ] ;
- local non-free = [ property-set.create [ $(requirements).base ]
- [ $(requirements).incidental ] ] ;
-
- local key = .rp.$(build-request)-$(non-free) ;
- if ! $($(key))
- {
- $(key) = [ common-properties2 $(build-request) $(non-free) ] ;
- }
- return [ $($(key)).add-raw $(free) ] ;
-}
-
-
-# Given a 'context' -- a set of already present properties, and 'requirements',
-# decide which extra properties should be applied to 'context'. For conditional
-# requirements, this means evaluating the condition. For indirect conditional
-# requirements, this means calling a rule. Ordinary requirements are always
-# applied.
-#
-# Handles the situation where evaluating one conditional requirement affects
-# conditions of another conditional requirements, such as:
-# <toolset>gcc:<variant>release <variant>release:<define>RELEASE
-#
-# If 'what' is 'refined' returns context refined with new requirements. If
-# 'what' is 'added' returns just the requirements to be applied.
-#
-rule evaluate-requirements ( requirements : context : what )
-{
- # Apply non-conditional requirements. It is possible that further
- # conditional requirement change a value set by non-conditional
- # requirements. For example:
- #
- # exe a : a.cpp : <threading>single <toolset>foo:<threading>multi ;
- #
- # I am not sure if this should be an error, or not, especially given that
- #
- # <threading>single
- #
- # might come from project's requirements.
-
- local unconditional = [ feature.expand [ $(requirements).non-conditional ] ]
- ;
-
- local raw = [ $(context).raw ] ;
- raw = [ property.refine $(raw) : $(unconditional) ] ;
-
- # We have collected properties that surely must be present in common
- # properties. We now try to figure out what other properties should be added
- # in order to satisfy rules (4)-(6) from the docs.
-
- local conditionals = [ $(requirements).conditional ] ;
- # The 'count' variable has one element for each conditional feature and for
- # each occurrence of '<indirect-conditional>' feature. It is used as a loop
- # counter: for each iteration of the loop before we remove one element and
- # the property set should stabilize before we are done. It is assumed that
- # #conditionals iterations should be enough for properties to propagate
- # along conditions in any direction.
- local count = $(conditionals) [ $(requirements).get <conditional> ]
- and-once-more ;
-
- local added-requirements ;
-
- local current = $(raw) ;
-
- # It is assumed that ordinary conditional requirements can not add
- # <conditional> properties (a.k.a. indirect conditional properties), and
- # that rules referred to by <conditional> properties can not add new
- # <conditional> properties. So the list of indirect conditionals does not
- # change.
- local indirect = [ $(requirements).get <conditional> ] ;
- indirect = [ MATCH ^@(.*) : $(indirect) ] ;
-
- local ok ;
- while $(count)
- {
- # Evaluate conditionals in context of current properties.
- local e = [ property.evaluate-conditionals-in-context $(conditionals) :
- $(current) ] ;
-
- # Evaluate indirect conditionals.
- for local i in $(indirect)
- {
- local t = [ current ] ;
- local p = [ $(t).project ] ;
- local new = [ indirect.call $(i) $(current) ] ;
- e += [ property.translate-paths $(new) : [ $(p).location ] ] ;
- }
-
- if $(e) = $(added-requirements)
- {
- # If we got the same result, we have found the final properties.
- count = ;
- ok = true ;
- }
- else
- {
- # Oops, conditional evaluation results have changed. Also 'current'
- # contains leftovers from a previous evaluation. Recompute 'current'
- # using initial properties and conditional requirements.
- added-requirements = $(e) ;
- current = [ property.refine $(raw) : [ feature.expand $(e) ] ] ;
- }
- count = $(count[2-]) ;
- }
- if ! $(ok)
- {
- import errors ;
- errors.error Can not evaluate conditional properties $(conditionals) ;
- }
-
- if $(what) = added
- {
- return [ property-set.create $(unconditional) $(added-requirements) ] ;
- }
- else if $(what) = refined
- {
- return [ property-set.create $(current) ] ;
- }
- else
- {
- import errors ;
- errors.error "Invalid value of the 'what' parameter." ;
- }
-}
-
-
-rule common-properties2 ( build-request requirements )
-{
- # This guarantees that default properties are present in the result, unless
- # they are overriden by some requirement. FIXME: There is a possibility that
- # we have added <foo>bar, which is composite and expands to <foo2>bar2, but
- # default value of <foo2> is not bar2, in which case it is not clear what to
- # do.
- #
- build-request = [ $(build-request).add-defaults ] ;
- # Features added by 'add-defaults' can be composite and expand to features
- # without default values -- which therefore have not been added yet. It
- # could be clearer/faster to expand only newly added properties but that is
- # not critical.
- build-request = [ $(build-request).expand ] ;
-
- return [ evaluate-requirements $(requirements) : $(build-request) :
- refined ] ;
-}
-
-
-rule push-target ( target )
-{
- .targets = $(target) $(.targets) ;
-}
-
-rule pop-target ( )
-{
- .targets = $(.targets[2-]) ;
-}
-
-# Return the metatarget that is currently being generated.
-rule current ( )
-{
- return $(.targets[1]) ;
-}
-
-
-# Implements the most standard way of constructing main target alternative from
-# sources. Allows sources to be either file or other main target and handles
-# generation of those dependency targets.
-#
-class basic-target : abstract-target
-{
- import build-request ;
- import build-system ;
- import "class" : new ;
- import feature ;
- import property ;
- import property-set ;
- import sequence ;
- import set ;
- import targets ;
- import virtual-target ;
-
- rule __init__ ( name : project : sources * : requirements * :
- default-build * : usage-requirements * )
- {
- abstract-target.__init__ $(name) : $(project) ;
-
- self.sources = $(sources) ;
- if ! $(requirements)
- {
- requirements = [ property-set.empty ] ;
- }
- self.requirements = $(requirements) ;
- if ! $(default-build)
- {
- default-build = [ property-set.empty ] ;
- }
- self.default-build = $(default-build) ;
- if ! $(usage-requirements)
- {
- usage-requirements = [ property-set.empty ] ;
- }
- self.usage-requirements = $(usage-requirements) ;
-
- if $(sources:G)
- {
- import errors : user-error : errors.user-error ;
- errors.user-error properties found "in" the 'sources' parameter
- "for" [ full-name ] ;
- }
- }
-
- rule always ( )
- {
- self.always = 1 ;
- }
-
- # Returns the list of abstract-targets which are used as sources. The extra
- # properties specified for sources are not represented. The only user for
- # this rule at the moment is the "--dump-tests" feature of the test system.
- #
- rule sources ( )
- {
- if ! $(self.source-targets)
- {
- for local s in $(self.sources)
- {
- self.source-targets += [ targets.resolve-reference $(s) :
- $(self.project) ] ;
- }
- }
- return $(self.source-targets) ;
- }
-
- rule requirements ( )
- {
- return $(self.requirements) ;
- }
-
- rule default-build ( )
- {
- return $(self.default-build) ;
- }
-
- # Returns the alternative condition for this alternative, if the condition
- # is satisfied by 'property-set'.
- #
- rule match ( property-set debug ? )
- {
- # The condition is composed of all base non-conditional properties. It
- # is not clear if we should expand 'self.requirements' or not. For one
- # thing, it would be nice to be able to put
- # <toolset>msvc-6.0
- # in requirements. On the other hand, if we have <variant>release as a
- # condition it does not make sense to require <optimization>full to be
- # in the build request just to select this variant.
- local bcondition = [ $(self.requirements).base ] ;
- local ccondition = [ $(self.requirements).conditional ] ;
- local condition = [ set.difference $(bcondition) : $(ccondition) ] ;
- if $(debug)
- {
- ECHO " next alternative: required properties:"
- $(condition:E=(empty)) ;
- }
-
- if $(condition) in [ $(property-set).raw ]
- {
- if $(debug)
- {
- ECHO " matched" ;
- }
- return $(condition) ;
- }
- else
- {
- if $(debug)
- {
- ECHO " not matched" ;
- }
- return no-match ;
- }
- }
-
- # Takes a target reference, which might be either target id or a dependency
- # property, and generates that target using 'property-set' as a build
- # request.
- #
- # The results are added to the variable called 'result-var'. Usage
- # requirements are added to the variable called 'usage-requirements-var'.
- #
- rule generate-dependencies ( dependencies * : property-set : result-var
- usage-requirements-var )
- {
- for local dependency in $(dependencies)
- {
- local grist = $(dependency:G) ;
- local id = $(dependency:G=) ;
- local result = [ targets.generate-from-reference $(id) :
- $(self.project) : $(property-set) ] ;
-
- $(result-var) += $(result[2-]:G=$(grist)) ;
- $(usage-requirements-var) += [ $(result[1]).raw ] ;
- }
- }
-
- # Determines final build properties, generates sources, and calls
- # 'construct'. This method should not be overridden.
- #
- rule generate ( property-set )
- {
- if [ modules.peek : .debug-building ]
- {
- ECHO ;
- local fn = [ full-name ] ;
- ECHO [ targets.indent ] "Building target '$(fn)'" ;
- targets.increase-indent ;
- ECHO [ targets.indent ] Build request: $(property-set)
- [ $(property-set).raw ] ;
- local cf = [ build-system.command-line-free-features ] ;
- ECHO [ targets.indent ] Command line free features: [ $(cf).raw ] ;
- ECHO [ targets.indent ] Target requirements:
- [ $(self.requirements).raw ] ;
- }
- targets.push-target $(__name__) ;
-
- # Apply free features from the command line. If user said
- # define=FOO
- # he most likely wants this define to be set for all compiles.
- # Make it before check for already built.
- property-set = [ $(property-set).refine
- [ build-system.command-line-free-features ] ] ;
-
- if ! $(self.generated.$(property-set))
- {
- local rproperties = [ targets.common-properties $(property-set)
- $(self.requirements) ] ;
-
- if [ modules.peek : .debug-building ]
- {
- ECHO ;
- ECHO [ targets.indent ] "Common properties: "
- [ $(rproperties).raw ] ;
- }
-
- if ( $(rproperties[1]) != "@error" ) && ( [ $(rproperties).get
- <build> ] != no )
- {
- local source-targets ;
- local properties = [ $(rproperties).non-dependency ] ;
- local usage-requirements ;
-
- generate-dependencies [ $(rproperties).dependency ] :
- $(rproperties) : properties usage-requirements ;
-
- generate-dependencies $(self.sources) : $(rproperties) :
- source-targets usage-requirements ;
-
- if [ modules.peek : .debug-building ]
- {
- ECHO ;
- ECHO [ targets.indent ] "Usage requirements for"
- $(self.name)": " $(usage-requirements) ;
- }
-
- rproperties = [ property-set.create $(properties)
- $(usage-requirements) ] ;
- usage-requirements = [ property-set.create $(usage-requirements)
- ] ;
-
- if [ modules.peek : .debug-building ]
- {
- ECHO [ targets.indent ] "Build properties: "
- [ $(rproperties).raw ] ;
- }
-
- local extra = [ $(rproperties).get <source> ] ;
- source-targets += $(extra:G=) ;
- # We might get duplicate sources, for example if we link to two
- # libraries having the same <library> usage requirement. Use
- # stable sort, since for some targets the order is important,
- # e.g. RUN_PY targets need a python source to come first.
- source-targets = [ sequence.unique $(source-targets) : stable ]
- ;
-
- local result = [ construct $(self.name) : $(source-targets) :
- $(rproperties) ] ;
-
- if $(result)
- {
- local gur = $(result[1]) ;
- result = $(result[2-]) ;
-
- if $(self.always)
- {
- for local t in $(result)
- {
- $(t).always ;
- }
- }
-
- local s = [ create-subvariant $(result)
- : [ virtual-target.recent-targets ]
- : $(property-set) : $(source-targets)
- : $(rproperties) : $(usage-requirements) ] ;
- virtual-target.clear-recent-targets ;
-
- local ur = [ compute-usage-requirements $(s) ] ;
- ur = [ $(ur).add $(gur) ] ;
- $(s).set-usage-requirements $(ur) ;
- if [ modules.peek : .debug-building ]
- {
- ECHO [ targets.indent ] "Usage requirements from"
- $(self.name)": " [ $(ur).raw ] ;
- }
-
- self.generated.$(property-set) = $(ur) $(result) ;
- }
- }
- else
- {
- if $(rproperties[1]) = "@error"
- {
- ECHO [ targets.indent ] "Skipping build of:" [ full-name ]
- "cannot compute common properties" ;
- }
- else if [ $(rproperties).get <build> ] = no
- {
- # If we just see <build>no, we cannot produce any reasonable
- # diagnostics. The code that adds this property is expected
- # to explain why a target is not built, for example using
- # the configure.log-component-configuration function.
- }
- else
- {
- ECHO [ targets.indent ] "Skipping build of: " [ full-name ]
- " unknown reason" ;
- }
-
- # We are here either because there has been an error computing
- # properties or there is <build>no in properties. In the latter
- # case we do not want any diagnostic. In the former case, we
- # need diagnostics. FIXME
-
- # If this target fails to build, add <build>no to properties to
- # cause any parent target to fail to build. Except that it
- # - does not work now, since we check for <build>no only in
- # common properties, but not in properties that came from
- # dependencies
- # - it is not clear if that is a good idea anyway. The alias
- # target, for example, should not fail to build if a
- # dependency fails.
- self.generated.$(property-set) = [ property-set.create <build>no
- ] ;
- }
- }
- else
- {
- if [ modules.peek : .debug-building ]
- {
- ECHO [ targets.indent ] "Already built" ;
- local ur = $(self.generated.$(property-set)) ;
- ur = $(ur[0]) ;
- targets.increase-indent ;
- ECHO [ targets.indent ] "Usage requirements from"
- $(self.name)": " [ $(ur).raw ] ;
- targets.decrease-indent ;
- }
- }
-
- targets.pop-target ;
- targets.decrease-indent ;
- return $(self.generated.$(property-set)) ;
- }
-
- # Given the set of generated targets, and refined build properties,
- # determines and sets appropriate usage requirements on those targets.
- #
- rule compute-usage-requirements ( subvariant )
- {
- local rproperties = [ $(subvariant).build-properties ] ;
- xusage-requirements = [ targets.evaluate-requirements
- $(self.usage-requirements) : $(rproperties) : added ] ;
-
- # We generate all dependency properties and add them, as well as their
- # usage requirements, to the result.
- local extra ;
- generate-dependencies [ $(xusage-requirements).dependency ] :
- $(rproperties) : extra extra ;
-
- local result = [ property-set.create
- [ $(xusage-requirements).non-dependency ] $(extra) ] ;
-
- # Propagate usage requirements we got from sources, except for the
- # <pch-header> and <pch-file> features.
- #
- # That feature specifies which pch file to use, and should apply only to
- # direct dependents. Consider:
- #
- # pch pch1 : ...
- # lib lib1 : ..... pch1 ;
- # pch pch2 :
- # lib lib2 : pch2 lib1 ;
- #
- # Here, lib2 should not get <pch-header> property from pch1.
- #
- # Essentially, when those two features are in usage requirements, they
- # are propagated only to direct dependents. We might need a more general
- # mechanism, but for now, only those two features are special.
- #
- # TODO - Actually there are more possible candidates like for instance
- # when listing static library X as a source for another static library.
- # Then static library X will be added as a <source> property to the
- # second library's usage requirements but those requirements should last
- # only up to the first executable or shared library that actually links
- # to it.
- local raw = [ $(subvariant).sources-usage-requirements ] ;
- raw = [ $(raw).raw ] ;
- raw = [ property.change $(raw) : <pch-header> ] ;
- raw = [ property.change $(raw) : <pch-file> ] ;
- return [ $(result).add [ property-set.create $(raw) ] ] ;
- }
-
- # Creates new subvariant instances for 'targets'.
- # 'root-targets' - virtual targets to be returned to dependants
- # 'all-targets' - virtual targets created while building this main target
- # 'build-request' - property-set instance with requested build properties
- #
- local rule create-subvariant ( root-targets * : all-targets * :
- build-request : sources * : rproperties : usage-requirements )
- {
- for local e in $(root-targets)
- {
- $(e).root true ;
- }
-
- # Process all virtual targets that will be created if this main target
- # is created.
- local s = [ new subvariant $(__name__) : $(build-request) : $(sources) :
- $(rproperties) : $(usage-requirements) : $(all-targets) ] ;
- for local v in $(all-targets)
- {
- if ! [ $(v).creating-subvariant ]
- {
- $(v).creating-subvariant $(s) ;
- }
- }
- return $(s) ;
- }
-
- # Constructs virtual targets for this abstract target and the dependency
- # graph. Returns a usage-requirements property-set and a list of virtual
- # targets. Should be overriden in derived classes.
- #
- rule construct ( name : source-targets * : properties * )
- {
- import errors : error : errors.error ;
- errors.error "method should be defined in derived classes" ;
- }
-}
-
-
-class typed-target : basic-target
-{
- import generators ;
-
- rule __init__ ( name : project : type : sources * : requirements * :
- default-build * : usage-requirements * )
- {
- basic-target.__init__ $(name) : $(project) : $(sources) :
- $(requirements) : $(default-build) : $(usage-requirements) ;
-
- self.type = $(type) ;
- }
-
- rule type ( )
- {
- return $(self.type) ;
- }
-
- rule construct ( name : source-targets * : property-set )
- {
- local r = [ generators.construct $(self.project) $(name:S=)
- : $(self.type)
- : [ property-set.create [ $(property-set).raw ]
- <main-target-type>$(self.type) ]
- : $(source-targets) : true ] ;
- if ! $(r)
- {
- ECHO "warn: Unable to construct" [ full-name ] ;
-
- # Are there any top-level generators for this type/property set.
- if ! [ generators.find-viable-generators $(self.type) :
- $(property-set) ]
- {
- ECHO "error: no generators were found for type '$(self.type)'" ;
- ECHO "error: and the requested properties" ;
- ECHO "error: make sure you've configured the needed tools" ;
- ECHO "See http://boost.org/boost-build2/doc/html/bbv2/advanced/configuration.html" ;
- EXIT "To debug this problem, try the --debug-generators option."
- ;
- }
- }
- return $(r) ;
- }
-}
-
-
-# Return the list of sources to use, if main target rule is invoked with
-# 'sources'. If there are any objects in 'sources', they are treated as main
-# target instances, and the name of such targets are adjusted to be
-# '<name_of_this_target>__<name_of_source_target>'. Such renaming is disabled if
-# a non-empty value is passed as the 'no-renaming' parameter.
-#
-rule main-target-sources ( sources * : main-target-name : no-renaming ? )
-{
- local result ;
- for local t in $(sources)
- {
- if [ class.is-instance $(t) ]
- {
- local name = [ $(t).name ] ;
- if ! $(no-renaming)
- {
- name = $(main-target-name)__$(name) ;
- $(t).rename $(name) ;
- }
- # Inline targets are not built by default.
- local p = [ $(t).project ] ;
- $(p).mark-target-as-explicit $(name) ;
- result += $(name) ;
- }
- else
- {
- result += $(t) ;
- }
- }
- return $(result) ;
-}
-
-
-# Returns the requirements to use when declaring a main target, obtained by
-# translating all specified property paths and refining project requirements
-# with the ones specified for the target.
-#
-rule main-target-requirements (
- specification * # Properties explicitly specified for the main target.
- : project # Project where the main target is to be declared.
-)
-{
- local requirements = [ property-set.refine-from-user-input
- [ $(project).get requirements ] : $(specification) :
- [ $(project).project-module ] : [ $(project).get location ] ] ;
- if $(requirements[1]) = "@error"
- {
- import errors ;
- errors.error "Conflicting requirements for target:" $(requirements) ;
- }
- return [ $(requirements).add [ toolset.requirements ] ] ;
-}
-
-
-# Returns the usage requirements to use when declaring a main target, which are
-# obtained by translating all specified property paths and adding project's
-# usage requirements.
-#
-rule main-target-usage-requirements (
- specification * # Use-properties explicitly specified for a main target.
- : project # Project where the main target is to be declared.
-)
-{
- local project-usage-requirements = [ $(project).get usage-requirements ] ;
-
- # We do not use 'refine-from-user-input' because:
- # - I am not sure if removing parent's usage requirements makes sense
- # - refining usage requirements is not needed, since usage requirements are
- # always free.
- local usage-requirements = [ property-set.create-from-user-input
- $(specification)
- : [ $(project).project-module ] [ $(project).get location ] ] ;
-
- return [ $(project-usage-requirements).add $(usage-requirements) ] ;
-}
-
-
-# Return the default build value to use when declaring a main target, which is
-# obtained by using the specified value if not empty and parent's default build
-# attribute otherwise.
-#
-rule main-target-default-build (
- specification * # Default build explicitly specified for a main target.
- : project # Project where the main target is to be declared.
-)
-{
- local result ;
- if $(specification)
- {
- result = $(specification) ;
- }
- else
- {
- result = [ $(project).get default-build ] ;
- }
- return [ property-set.create-with-validation $(result) ] ;
-}
-
-
-# Registers the specified target as a main target alternative and returns it.
-#
-rule main-target-alternative ( target )
-{
- local ptarget = [ $(target).project ] ;
- $(ptarget).add-alternative $(target) ;
- return $(target) ;
-}
-
-
-# Creates a metatarget with the specified properties, using 'klass' as the
-# class. The 'name', 'sources', 'requirements', 'default-build' and
-# 'usage-requirements' are assumed to be in the form specified by the user in
-# the Jamfile corresponding to 'project'.
-#
-rule create-metatarget ( klass : project : name : sources * : requirements * :
- default-build * : usage-requirements * )
-{
- return [ targets.main-target-alternative [ new $(klass) $(name) : $(project)
- : [ targets.main-target-sources $(sources) : $(name) ]
- : [ targets.main-target-requirements $(requirements) : $(project) ]
- : [ targets.main-target-default-build $(default-build) : $(project) ]
- : [ targets.main-target-usage-requirements $(usage-requirements) :
- $(project) ] ] ] ;
-}
-
-
-# Creates a typed-target with the specified properties. The 'name', 'sources',
-# 'requirements', 'default-build' and 'usage-requirements' are assumed to be in
-# the form specified by the user in the Jamfile corresponding to 'project'.
-#
-rule create-typed-target ( type : project : name : sources * : requirements * :
- default-build * : usage-requirements * )
-{
- return [ targets.main-target-alternative [ new typed-target $(name) :
- $(project) : $(type)
- : [ targets.main-target-sources $(sources) : $(name) ]
- : [ targets.main-target-requirements $(requirements) : $(project) ]
- : [ targets.main-target-default-build $(default-build) : $(project) ]
- : [ targets.main-target-usage-requirements $(usage-requirements) :
- $(project) ] ] ] ;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/boost-build/build/toolset.jam
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/boost-build/build/toolset.jam b/ext/kenlm/jam-files/boost-build/build/toolset.jam
deleted file mode 100644
index 6e5b980..0000000
--- a/ext/kenlm/jam-files/boost-build/build/toolset.jam
+++ /dev/null
@@ -1,582 +0,0 @@
-# Copyright 2003 Dave Abrahams
-# Copyright 2005 Rene Rivera
-# Copyright 2002, 2003, 2004, 2005, 2006 Vladimir Prus
-# Distributed under the Boost Software License, Version 1.0.
-# (See accompanying file LICENSE_1_0.txt or copy at
-# http://www.boost.org/LICENSE_1_0.txt)
-
-# Support for toolset definition.
-
-import errors ;
-import feature ;
-import generators ;
-import numbers ;
-import path ;
-import property ;
-import regex ;
-import sequence ;
-import set ;
-import property-set ;
-
-
-.flag-no = 1 ;
-
-.ignore-requirements = ;
-
-# This is used only for testing, to make sure we do not get random extra
-# elements in paths.
-if --ignore-toolset-requirements in [ modules.peek : ARGV ]
-{
- .ignore-requirements = 1 ;
-}
-
-
-# Initializes an additional toolset-like module. First load the 'toolset-module'
-# and then calls its 'init' rule with trailing arguments.
-#
-rule using ( toolset-module : * )
-{
- import $(toolset-module) ;
- $(toolset-module).init $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9)
- ;
-}
-
-
-# Expands subfeatures in each property sets, e.g. '<toolset>gcc-3.2' will be
-# converted to '<toolset>gcc/<toolset-version>3.2'.
-#
-local rule normalize-condition ( property-sets * )
-{
- local result ;
- for local p in $(property-sets)
- {
- local split = [ feature.split $(p) ] ;
- local expanded = [ feature.expand-subfeatures [ feature.split $(p) ] ] ;
- result += $(expanded:J=/) ;
- }
- return $(result) ;
-}
-
-
-# Specifies if the 'flags' rule should check that the invoking module is the
-# same as the module we are setting the flag for. 'v' can be either 'checked' or
-# 'unchecked'. Subsequent call to 'pop-checking-for-flags-module' will restore
-# the setting that was in effect before calling this rule.
-#
-rule push-checking-for-flags-module ( v )
-{
- .flags-module-checking = $(v) $(.flags-module-checking) ;
-}
-
-rule pop-checking-for-flags-module ( )
-{
- .flags-module-checking = $(.flags-module-checking[2-]) ;
-}
-
-
-# Specifies the flags (variables) that must be set on targets under certain
-# conditions, described by arguments.
-#
-rule flags (
- rule-or-module # If contains a dot, should be a rule name. The flags will
- # be applied when that rule is used to set up build
- # actions.
- #
- # If does not contain dot, should be a module name. The
- # flag will be applied for all rules in that module. If
- # module for rule is different from the calling module, an
- # error is issued.
-
- variable-name # Variable that should be set on target.
- condition * : # A condition when this flag should be applied. Should be a
- # set of property sets. If one of those property sets is
- # contained in the build properties, the flag will be used.
- # Implied values are not allowed: "<toolset>gcc" should be
- # used, not just "gcc". Subfeatures, like in
- # "<toolset>gcc-3.2" are allowed. If left empty, the flag
- # will be used unconditionally.
- #
- # Propery sets may use value-less properties ('<a>' vs.
- # '<a>value') to match absent properties. This allows to
- # separately match:
- #
- # <architecture>/<address-model>64
- # <architecture>ia64/<address-model>
- #
- # Where both features are optional. Without this syntax
- # we would be forced to define "default" values.
-
- values * : # The value to add to variable. If <feature> is specified,
- # then the value of 'feature' will be added.
- unchecked ? # If value 'unchecked' is passed, will not test that flags
- # are set for the calling module.
- : hack-hack ? # For
- # flags rule OPTIONS <cxx-abi> : -model ansi
- # Treat <cxx-abi> as condition
- # FIXME: ugly hack.
-)
-{
- local caller = [ CALLER_MODULE ] ;
- if ! [ MATCH ".*([.]).*" : $(rule-or-module) ]
- && [ MATCH "(Jamfile<.*)" : $(caller) ]
- {
- # Unqualified rule name, used inside Jamfile. Most likely used with
- # 'make' or 'notfile' rules. This prevents setting flags on the entire
- # Jamfile module (this will be considered as rule), but who cares?
- # Probably, 'flags' rule should be split into 'flags' and
- # 'flags-on-module'.
- rule-or-module = $(caller).$(rule-or-module) ;
- }
- else
- {
- local module_ = [ MATCH "([^.]*).*" : $(rule-or-module) ] ;
- if $(unchecked) != unchecked
- && $(.flags-module-checking[1]) != unchecked
- && $(module_) != $(caller)
- {
- errors.error "Module $(caller) attempted to set flags for module $(module_)" ;
- }
- }
-
- if $(condition) && ! $(condition:G=) && ! $(hack-hack)
- {
- # We have condition in the form '<feature>', that is, without value.
- # That is an older syntax:
- # flags gcc.link RPATH <dll-path> ;
- # for compatibility, convert it to
- # flags gcc.link RPATH : <dll-path> ;
- values = $(condition) ;
- condition = ;
- }
-
- if $(condition)
- {
- property.validate-property-sets $(condition) ;
- condition = [ normalize-condition $(condition) ] ;
- }
-
- add-flag $(rule-or-module) : $(variable-name) : $(condition) : $(values) ;
-}
-
-
-# Adds a new flag setting with the specified values. Does no checking.
-#
-local rule add-flag ( rule-or-module : variable-name : condition * : values * )
-{
- .$(rule-or-module).flags += $(.flag-no) ;
-
- # Store all flags for a module.
- local module_ = [ MATCH "([^.]*).*" : $(rule-or-module) ] ;
- .module-flags.$(module_) += $(.flag-no) ;
- # Store flag-no -> rule-or-module mapping.
- .rule-or-module.$(.flag-no) = $(rule-or-module) ;
-
- .$(rule-or-module).variable.$(.flag-no) += $(variable-name) ;
- .$(rule-or-module).values.$(.flag-no) += $(values) ;
- .$(rule-or-module).condition.$(.flag-no) += $(condition) ;
-
- .flag-no = [ numbers.increment $(.flag-no) ] ;
-}
-
-
-# Returns the first element of 'property-sets' which is a subset of
-# 'properties' or an empty list if no such element exists.
-#
-rule find-property-subset ( property-sets * : properties * )
-{
- # Cut property values off.
- local prop-keys = $(properties:G) ;
-
- local result ;
- for local s in $(property-sets)
- {
- if ! $(result)
- {
- # Handle value-less properties like '<architecture>' (compare with
- # '<architecture>x86').
-
- local set = [ feature.split $(s) ] ;
-
- # Find the set of features that
- # - have no property specified in required property set
- # - are omitted in the build property set.
- local default-props ;
- for local i in $(set)
- {
- # If $(i) is a value-less property it should match default value
- # of an optional property. See the first line in the example
- # below:
- #
- # property set properties result
- # <a> <b>foo <b>foo match
- # <a> <b>foo <a>foo <b>foo no match
- # <a>foo <b>foo <b>foo no match
- # <a>foo <b>foo <a>foo <b>foo match
- if ! ( $(i:G=) || ( $(i:G) in $(prop-keys) ) )
- {
- default-props += $(i) ;
- }
- }
-
- if $(set) in $(properties) $(default-props)
- {
- result = $(s) ;
- }
- }
- }
- return $(result) ;
-}
-
-
-# Returns a value to be added to some flag for some target based on the flag's
-# value definition and the given target's property set.
-#
-rule handle-flag-value ( value * : properties * )
-{
- local result ;
- if $(value:G)
- {
- local matches = [ property.select $(value) : $(properties) ] ;
- for local p in $(matches)
- {
- local att = [ feature.attributes $(p:G) ] ;
- if dependency in $(att)
- {
- # The value of a dependency feature is a target and needs to be
- # actualized.
- result += [ $(p:G=).actualize ] ;
- }
- else if path in $(att) || free in $(att)
- {
- local values ;
- # Treat features with && in the value specially -- each
- # &&-separated element is considered a separate value. This is
- # needed to handle searched libraries or include paths, which
- # may need to be in a specific order.
- if ! [ MATCH (&&) : $(p:G=) ]
- {
- values = $(p:G=) ;
- }
- else
- {
- values = [ regex.split $(p:G=) "&&" ] ;
- }
- if path in $(att)
- {
- result += [ sequence.transform path.native : $(values) ] ;
- }
- else
- {
- result += $(values) ;
- }
- }
- else
- {
- result += $(p:G=) ;
- }
- }
- }
- else
- {
- result += $(value) ;
- }
- return $(result) ;
-}
-
-
-# Given a rule name and a property set, returns a list of interleaved variables
-# names and values which must be set on targets for that rule/property-set
-# combination.
-#
-rule set-target-variables-aux ( rule-or-module : property-set )
-{
- local result ;
- properties = [ $(property-set).raw ] ;
- for local f in $(.$(rule-or-module).flags)
- {
- local variable = $(.$(rule-or-module).variable.$(f)) ;
- local condition = $(.$(rule-or-module).condition.$(f)) ;
- local values = $(.$(rule-or-module).values.$(f)) ;
-
- if ! $(condition) ||
- [ find-property-subset $(condition) : $(properties) ]
- {
- local processed ;
- for local v in $(values)
- {
- # The value might be <feature-name> so needs special treatment.
- processed += [ handle-flag-value $(v) : $(properties) ] ;
- }
- for local r in $(processed)
- {
- result += $(variable) $(r) ;
- }
- }
- }
-
- # Strip away last dot separated part and recurse.
- local next = [ MATCH ^(.+)\\.([^\\.])* : $(rule-or-module) ] ;
- if $(next)
- {
- result += [ set-target-variables-aux $(next[1]) : $(property-set) ] ;
- }
- return $(result) ;
-}
-
-rule relevant-features ( rule-or-module )
-{
- local result ;
- if ! $(.relevant-features.$(rule-or-module))
- {
- for local f in $(.$(rule-or-module).flags)
- {
- local condition = $(.$(rule-or-module).condition.$(f)) ;
- local values = $(.$(rule-or-module).values.$(f)) ;
-
- for local c in $(condition)
- {
- for local p in [ feature.split $(c) ]
- {
- if $(p:G)
- {
- result += $(p:G) ;
- }
- else
- {
- local temp = [ feature.expand-subfeatures $(p) ] ;
- result += $(temp:G) ;
- }
- }
- }
-
- for local v in $(values)
- {
- if $(v:G)
- {
- result += $(v:G) ;
- }
- }
- }
-
- # Strip away last dot separated part and recurse.
- local next = [ MATCH ^(.+)\\.([^\\.])* : $(rule-or-module) ] ;
- if $(next)
- {
- result += [ relevant-features $(next[1]) ] ;
- }
- result = [ sequence.unique $(result) ] ;
- if $(result[1]) = ""
- {
- result = $(result) ;
- }
- .relevant-features.$(rule-or-module) = $(result) ;
- return $(result) ;
- }
- else
- {
- return $(.relevant-features.$(rule-or-module)) ;
- }
-}
-
-rule filter-property-set ( rule-or-module : property-set )
-{
- local key = .filtered.property-set.$(rule-or-module).$(property-set) ;
- if ! $($(key))
- {
- local relevant = [ relevant-features $(rule-or-module) ] ;
- local result ;
- for local p in [ $(property-set).raw ]
- {
- if $(p:G) in $(relevant)
- {
- result += $(p) ;
- }
- }
- $(key) = [ property-set.create $(result) ] ;
- }
- return $($(key)) ;
-}
-
-rule set-target-variables ( rule-or-module targets + : property-set )
-{
- property-set = [ filter-property-set $(rule-or-module) : $(property-set) ] ;
- local key = .stv.$(rule-or-module).$(property-set) ;
- local settings = $($(key)) ;
- if ! $(settings)
- {
- settings = [ set-target-variables-aux $(rule-or-module) :
- $(property-set) ] ;
-
- if ! $(settings)
- {
- settings = none ;
- }
- $(key) = $(settings) ;
- }
-
- if $(settings) != none
- {
- local var-name = ;
- for local name-or-value in $(settings)
- {
- if $(var-name)
- {
- $(var-name) on $(targets) += $(name-or-value) ;
- var-name = ;
- }
- else
- {
- var-name = $(name-or-value) ;
- }
- }
- }
-}
-
-
-# Make toolset 'toolset', defined in a module of the same name, inherit from
-# 'base'.
-# 1. The 'init' rule from 'base' is imported into 'toolset' with full name.
-# Another 'init' is called, which forwards to the base one.
-# 2. All generators from 'base' are cloned. The ids are adjusted and <toolset>
-# property in requires is adjusted too.
-# 3. All flags are inherited.
-# 4. All rules are imported.
-#
-rule inherit ( toolset : base )
-{
- import $(base) ;
- inherit-generators $(toolset) : $(base) ;
- inherit-flags $(toolset) : $(base) ;
- inherit-rules $(toolset) : $(base) ;
-}
-
-
-rule inherit-generators ( toolset properties * : base : generators-to-ignore * )
-{
- properties ?= <toolset>$(toolset) ;
- local base-generators = [ generators.generators-for-toolset $(base) ] ;
- for local g in $(base-generators)
- {
- local id = [ $(g).id ] ;
-
- if ! $(id) in $(generators-to-ignore)
- {
- # Some generator names have multiple periods in their name, so
- # $(id:B=$(toolset)) does not generate the right new-id name. E.g.
- # if id = gcc.compile.c++ then $(id:B=darwin) = darwin.c++, which is
- # not what we want. Manually parse the base and suffix. If there is
- # a better way to do this, I would love to see it. See also the
- # register() rule in the generators module.
- local base = $(id) ;
- local suffix = "" ;
- while $(base:S)
- {
- suffix = $(base:S)$(suffix) ;
- base = $(base:B) ;
- }
- local new-id = $(toolset)$(suffix) ;
-
- generators.register [ $(g).clone $(new-id) : $(properties) ] ;
- }
- }
-}
-
-
-# Brings all flag definitions from the 'base' toolset into the 'toolset'
-# toolset. Flag definitions whose conditions make use of properties in
-# 'prohibited-properties' are ignored. Do not confuse property and feature, for
-# example <debug-symbols>on and <debug-symbols>off, so blocking one of them does
-# not block the other one.
-#
-# The flag conditions are not altered at all, so if a condition includes a name,
-# or version of a base toolset, it will not ever match the inheriting toolset.
-# When such flag settings must be inherited, define a rule in base toolset
-# module and call it as needed.
-#
-rule inherit-flags ( toolset : base : prohibited-properties * : prohibited-vars * )
-{
- for local f in $(.module-flags.$(base))
- {
- local rule-or-module = $(.rule-or-module.$(f)) ;
- if ( [ set.difference
- $(.$(rule-or-module).condition.$(f)) :
- $(prohibited-properties) ]
- || ! $(.$(rule-or-module).condition.$(f))
- ) && ( ! $(.$(rule-or-module).variable.$(f)) in $(prohibited-vars) )
- {
- local rule_ = [ MATCH "[^.]*\.(.*)" : $(rule-or-module) ] ;
- local new-rule-or-module ;
- if $(rule_)
- {
- new-rule-or-module = $(toolset).$(rule_) ;
- }
- else
- {
- new-rule-or-module = $(toolset) ;
- }
-
- add-flag
- $(new-rule-or-module)
- : $(.$(rule-or-module).variable.$(f))
- : $(.$(rule-or-module).condition.$(f))
- : $(.$(rule-or-module).values.$(f)) ;
- }
- }
-}
-
-
-rule inherit-rules ( toolset : base : localize ? )
-{
- # It appears that "action" creates a local rule.
- local base-generators = [ generators.generators-for-toolset $(base) ] ;
- local rules ;
- for local g in $(base-generators)
- {
- rules += [ MATCH "[^.]*\.(.*)" : [ $(g).rule-name ] ] ;
- }
- rules = [ sequence.unique $(rules) ] ;
- IMPORT $(base) : $(rules) : $(toolset) : $(rules) : $(localize) ;
- IMPORT $(toolset) : $(rules) : : $(toolset).$(rules) ;
-}
-
-.requirements = [ property-set.empty ] ;
-
-# Return the list of global 'toolset requirements'. Those requirements will be
-# automatically added to the requirements of any main target.
-#
-rule requirements ( )
-{
- return $(.requirements) ;
-}
-
-
-# Adds elements to the list of global 'toolset requirements'. The requirements
-# will be automatically added to the requirements for all main targets, as if
-# they were specified literally. For best results, all requirements added should
-# be conditional or indirect conditional.
-#
-rule add-requirements ( requirements * )
-{
- if ! $(.ignore-requirements)
- {
- requirements = [ property.translate-indirect $(specification) : [ CALLER_MODULE ] ] ;
- requirements = [ property.expand-subfeatures-in-conditions $(requirements) ] ;
- requirements = [ property.make $(requirements) ] ;
- .requirements = [ $(.requirements).add-raw $(requirements) ] ;
- }
-}
-
-
-rule __test__ ( )
-{
- import assert ;
- local p = <b>0 <c>1 <d>2 <e>3 <f>4 ;
- assert.result <c>1/<d>2/<e>3 : find-property-subset <c>1/<d>2/<e>3 <a>0/<b>0/<c>1 <d>2/<e>5 <a>9 : $(p) ;
- assert.result : find-property-subset <a>0/<b>0/<c>9/<d>9/<e>5 <a>9 : $(p) ;
-
- local p-set = <a>/<b> <a>0/<b> <a>/<b>1 <a>0/<b>1 ;
- assert.result <a>/<b> : find-property-subset $(p-set) : ;
- assert.result <a>0/<b> : find-property-subset $(p-set) : <a>0 <c>2 ;
- assert.result <a>/<b>1 : find-property-subset $(p-set) : <b>1 <c>2 ;
- assert.result <a>0/<b>1 : find-property-subset $(p-set) : <a>0 <b>1 ;
-}
[26/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/execnt.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/execnt.c b/ext/kenlm/jam-files/engine/execnt.c
deleted file mode 100644
index ca89bd0..0000000
--- a/ext/kenlm/jam-files/engine/execnt.c
+++ /dev/null
@@ -1,1232 +0,0 @@
-/*
- * Copyright 1993, 1995 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Copyright 2007 Rene Rivera.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * execnt.c - execute a shell command on Windows NT
- *
- * If $(JAMSHELL) is defined, uses that to formulate the actual command. The
- * default is: cmd.exe /Q/C
- *
- * In $(JAMSHELL), % expands to the command string and ! expands to the slot
- * number (starting at 1) for multiprocess (-j) invocations. If $(JAMSHELL) does
- * not include a %, it is tacked on as the last argument.
- *
- * Each $(JAMSHELL) placeholder must be specified as a separate individual
- * element in a jam variable value.
- *
- * Do not just set JAMSHELL to cmd.exe - it will not work!
- *
- * External routines:
- * exec_check() - preprocess and validate the command
- * exec_cmd() - launch an async command execution
- * exec_wait() - wait for any of the async command processes to terminate
- *
- * Internal routines:
- * filetime_to_seconds() - Windows FILETIME --> number of seconds conversion
- */
-
-#include "jam.h"
-#ifdef USE_EXECNT
-#include "execcmd.h"
-
-#include "lists.h"
-#include "output.h"
-#include "pathsys.h"
-#include "string.h"
-
-#include <assert.h>
-#include <ctype.h>
-#include <errno.h>
-#include <time.h>
-
-#define WIN32_LEAN_AND_MEAN
-#include <windows.h>
-#include <process.h>
-#include <tlhelp32.h>
-
-
-/* get the maximum shell command line length according to the OS */
-static int maxline();
-/* valid raw command string length */
-static long raw_command_length( char const * command );
-/* add two 64-bit unsigned numbers, h1l1 and h2l2 */
-static FILETIME add_64(
- unsigned long h1, unsigned long l1,
- unsigned long h2, unsigned long l2 );
-/* */
-static FILETIME add_FILETIME( FILETIME t1, FILETIME t2 );
-/* */
-static FILETIME negate_FILETIME( FILETIME t );
-/* record the timing info for the process */
-static void record_times( HANDLE const, timing_info * const );
-/* calc the current running time of an *active* process */
-static double running_time( HANDLE const );
-/* terminate the given process, after terminating all its children first */
-static void kill_process_tree( DWORD const procesdId, HANDLE const );
-/* waits for a command to complete or time out */
-static int try_wait( int const timeoutMillis );
-/* reads any pending output for running commands */
-static void read_output();
-/* checks if a command ran out of time, and kills it */
-static int try_kill_one();
-/* is the first process a parent (direct or indirect) to the second one */
-static int is_parent_child( DWORD const parent, DWORD const child );
-/* */
-static void close_alert( PROCESS_INFORMATION const * const );
-/* close any alerts hanging around */
-static void close_alerts();
-/* prepare a command file to be executed using an external shell */
-static char const * prepare_command_file( string const * command, int slot );
-/* invoke the actual external process using the given command line */
-static void invoke_cmd( char const * const command, int const slot );
-/* find a free slot in the running commands table */
-static int get_free_cmdtab_slot();
-/* put together the final command string we are to run */
-static void string_new_from_argv( string * result, char const * const * argv );
-/* frees and renews the given string */
-static void string_renew( string * const );
-/* reports the last failed Windows API related error message */
-static void reportWindowsError( char const * const apiName );
-/* closes a Windows HANDLE and resets its variable to 0. */
-static void closeWinHandle( HANDLE * const handle );
-
-/* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */
-
-/* CreateProcessA() Windows API places a limit of 32768 characters (bytes) on
- * the allowed command-line length, including a trailing Unicode (2-byte)
- * nul-terminator character.
- */
-#define MAX_RAW_COMMAND_LENGTH 32766
-
-/* We hold handles for pipes used to communicate with child processes in two
- * element arrays indexed as follows.
- */
-#define EXECCMD_PIPE_READ 0
-#define EXECCMD_PIPE_WRITE 1
-
-static int intr_installed;
-
-
-/* The list of commands we run. */
-static struct
-{
- /* Temporary command file used to execute the action when needed. */
- string command_file[ 1 ];
-
- /* Pipes for communicating with the child process. Parent reads from (0),
- * child writes to (1).
- */
- HANDLE pipe_out[ 2 ];
- HANDLE pipe_err[ 2 ];
-
- string buffer_out[ 1 ]; /* buffer to hold stdout, if any */
- string buffer_err[ 1 ]; /* buffer to hold stderr, if any */
-
- PROCESS_INFORMATION pi; /* running process information */
-
- /* Function called when the command completes. */
- ExecCmdCallback func;
-
- /* Opaque data passed back to the 'func' callback. */
- void * closure;
-}
-cmdtab[ MAXJOBS ] = { { 0 } };
-
-
-/*
- * Execution unit tests.
- */
-
-void execnt_unit_test()
-{
-#if !defined( NDEBUG )
- /* vc6 preprocessor is broken, so assert with these strings gets confused.
- * Use a table instead.
- */
- {
- typedef struct test { char * command; int result; } test;
- test tests[] = {
- { "", 0 },
- { " ", 0 },
- { "x", 1 },
- { "\nx", 1 },
- { "x\n", 1 },
- { "\nx\n", 1 },
- { "\nx \n", 2 },
- { "\nx \n ", 2 },
- { " \n\t\t\v\r\r\n \t x \v \t\t\r\n\n\n \n\n\v\t", 8 },
- { "x\ny", -1 },
- { "x\n\n y", -1 },
- { "echo x > foo.bar", -1 },
- { "echo x < foo.bar", -1 },
- { "echo x | foo.bar", -1 },
- { "echo x \">\" foo.bar", 18 },
- { "echo x '<' foo.bar", 18 },
- { "echo x \"|\" foo.bar", 18 },
- { "echo x \\\">\\\" foo.bar", -1 },
- { "echo x \\\"<\\\" foo.bar", -1 },
- { "echo x \\\"|\\\" foo.bar", -1 },
- { "\"echo x > foo.bar\"", 18 },
- { "echo x \"'\"<' foo.bar", -1 },
- { "echo x \\\\\"<\\\\\" foo.bar", 22 },
- { "echo x \\x\\\"<\\\\\" foo.bar", -1 },
- { 0 } };
- test const * t;
- for ( t = tests; t->command; ++t )
- assert( raw_command_length( t->command ) == t->result );
- }
-
- {
- int const length = maxline() + 9;
- char * const cmd = (char *)BJAM_MALLOC_ATOMIC( length + 1 );
- memset( cmd, 'x', length );
- cmd[ length ] = 0;
- assert( raw_command_length( cmd ) == length );
- BJAM_FREE( cmd );
- }
-#endif
-}
-
-
-/*
- * exec_check() - preprocess and validate the command
- */
-
-int exec_check
-(
- string const * command,
- LIST * * pShell,
- int * error_length,
- int * error_max_length
-)
-{
- /* Default shell does nothing when triggered with an empty or a
- * whitespace-only command so we simply skip running it in that case. We
- * still pass them on to non-default shells as we do not really know what
- * they are going to do with such commands.
- */
- if ( list_empty( *pShell ) )
- {
- char const * s = command->value;
- while ( isspace( *s ) ) ++s;
- if ( !*s )
- return EXEC_CHECK_NOOP;
- }
-
- /* Check prerequisites for executing raw commands. */
- if ( is_raw_command_request( *pShell ) )
- {
- int const raw_cmd_length = raw_command_length( command->value );
- if ( raw_cmd_length < 0 )
- {
- /* Invalid characters detected - fallback to default shell. */
- list_free( *pShell );
- *pShell = L0;
- }
- else if ( raw_cmd_length > MAX_RAW_COMMAND_LENGTH )
- {
- *error_length = raw_cmd_length;
- *error_max_length = MAX_RAW_COMMAND_LENGTH;
- return EXEC_CHECK_TOO_LONG;
- }
- else
- return raw_cmd_length ? EXEC_CHECK_OK : EXEC_CHECK_NOOP;
- }
-
- /* Now we know we are using an external shell. Note that there is no need to
- * check for too long command strings when using an external shell since we
- * use a command file and assume no one is going to set up a JAMSHELL format
- * string longer than a few hundred bytes at most which should be well under
- * the total command string limit. Should someone actually construct such a
- * JAMSHELL value it will get reported as an 'invalid parameter'
- * CreateProcessA() Windows API failure which seems like a good enough
- * result for such intentional mischief.
- */
-
- /* Check for too long command lines. */
- return check_cmd_for_too_long_lines( command->value, maxline(),
- error_length, error_max_length );
-}
-
-
-/*
- * exec_cmd() - launch an async command execution
- *
- * We assume exec_check() already verified that the given command can have its
- * command string constructed as requested.
- */
-
-void exec_cmd
-(
- string const * cmd_orig,
- ExecCmdCallback func,
- void * closure,
- LIST * shell
-)
-{
- int const slot = get_free_cmdtab_slot();
- int const is_raw_cmd = is_raw_command_request( shell );
- string cmd_local[ 1 ];
-
- /* Initialize default shell - anything more than /Q/C is non-portable. */
- static LIST * default_shell;
- if ( !default_shell )
- default_shell = list_new( object_new( "cmd.exe /Q/C" ) );
-
- /* Specifying no shell means requesting the default shell. */
- if ( list_empty( shell ) )
- shell = default_shell;
-
- if ( DEBUG_EXECCMD )
- if ( is_raw_cmd )
- printf( "Executing raw command directly\n" );
- else
- {
- printf( "Executing using a command file and the shell: " );
- list_print( shell );
- printf( "\n" );
- }
-
- /* If we are running a raw command directly - trim its leading whitespaces
- * as well as any trailing all-whitespace lines but keep any trailing
- * whitespace in the final/only line containing something other than
- * whitespace).
- */
- if ( is_raw_cmd )
- {
- char const * start = cmd_orig->value;
- char const * p = cmd_orig->value + cmd_orig->size;
- char const * end = p;
- while ( isspace( *start ) ) ++start;
- while ( p > start && isspace( p[ -1 ] ) )
- if ( *--p == '\n' )
- end = p;
- string_new( cmd_local );
- string_append_range( cmd_local, start, end );
- assert( cmd_local->size == raw_command_length( cmd_orig->value ) );
- }
- /* If we are not running a raw command directly, prepare a command file to
- * be executed using an external shell and the actual command string using
- * that command file.
- */
- else
- {
- char const * const cmd_file = prepare_command_file( cmd_orig, slot );
- char const * argv[ MAXARGC + 1 ]; /* +1 for NULL */
- argv_from_shell( argv, shell, cmd_file, slot );
- string_new_from_argv( cmd_local, argv );
- }
-
- /* Catch interrupts whenever commands are running. */
- if ( !intr_installed )
- {
- intr_installed = 1;
- signal( SIGINT, onintr );
- }
-
- /* Save input data into the selected running commands table slot. */
- cmdtab[ slot ].func = func;
- cmdtab[ slot ].closure = closure;
-
- /* Invoke the actual external process using the constructed command line. */
- invoke_cmd( cmd_local->value, slot );
-
- /* Free our local command string copy. */
- string_free( cmd_local );
-}
-
-
-/*
- * exec_wait() - wait for any of the async command processes to terminate
- *
- * Wait and drive at most one execution completion, while processing the I/O for
- * all ongoing commands.
- */
-
-void exec_wait()
-{
- int i = -1;
- int exit_reason; /* reason why a command completed */
-
- /* Wait for a command to complete, while snarfing up any output. */
- while ( 1 )
- {
- /* Check for a complete command, briefly. */
- i = try_wait( 500 );
- /* Read in the output of all running commands. */
- read_output();
- /* Close out pending debug style dialogs. */
- close_alerts();
- /* Process the completed command we found. */
- if ( i >= 0 ) { exit_reason = EXIT_OK; break; }
- /* Check if a command ran out of time. */
- i = try_kill_one();
- if ( i >= 0 ) { exit_reason = EXIT_TIMEOUT; break; }
- }
-
- /* We have a command... process it. */
- {
- DWORD exit_code;
- timing_info time;
- int rstat;
-
- /* The time data for the command. */
- record_times( cmdtab[ i ].pi.hProcess, &time );
-
- /* Removed the used temporary command file. */
- if ( cmdtab[ i ].command_file->size )
- unlink( cmdtab[ i ].command_file->value );
-
- /* Find out the process exit code. */
- GetExitCodeProcess( cmdtab[ i ].pi.hProcess, &exit_code );
-
- /* The dispossition of the command. */
- if ( interrupted() )
- rstat = EXEC_CMD_INTR;
- else if ( exit_code )
- rstat = EXEC_CMD_FAIL;
- else
- rstat = EXEC_CMD_OK;
-
- /* Call the callback, may call back to jam rule land. */
- (*cmdtab[ i ].func)( cmdtab[ i ].closure, rstat, &time,
- cmdtab[ i ].buffer_out->value, cmdtab[ i ].buffer_err->value,
- exit_reason );
-
- /* Clean up our child process tracking data. No need to clear the
- * temporary command file name as it gets reused.
- */
- closeWinHandle( &cmdtab[ i ].pi.hProcess );
- closeWinHandle( &cmdtab[ i ].pi.hThread );
- closeWinHandle( &cmdtab[ i ].pipe_out[ EXECCMD_PIPE_READ ] );
- closeWinHandle( &cmdtab[ i ].pipe_out[ EXECCMD_PIPE_WRITE ] );
- closeWinHandle( &cmdtab[ i ].pipe_err[ EXECCMD_PIPE_READ ] );
- closeWinHandle( &cmdtab[ i ].pipe_err[ EXECCMD_PIPE_WRITE ] );
- string_renew( cmdtab[ i ].buffer_out );
- string_renew( cmdtab[ i ].buffer_err );
- }
-}
-
-
-/* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */
-
-/*
- * Invoke the actual external process using the given command line. Track the
- * process in our running commands table.
- */
-
-static void invoke_cmd( char const * const command, int const slot )
-{
- SECURITY_ATTRIBUTES sa = { sizeof( SECURITY_ATTRIBUTES ), 0, 0 };
- SECURITY_DESCRIPTOR sd;
- STARTUPINFO si = { sizeof( STARTUPINFO ), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0 };
-
- /* Init the security data. */
- InitializeSecurityDescriptor( &sd, SECURITY_DESCRIPTOR_REVISION );
- SetSecurityDescriptorDacl( &sd, TRUE, NULL, FALSE );
- sa.lpSecurityDescriptor = &sd;
- sa.bInheritHandle = TRUE;
-
- /* Create pipes for communicating with the child process. */
- if ( !CreatePipe( &cmdtab[ slot ].pipe_out[ EXECCMD_PIPE_READ ],
- &cmdtab[ slot ].pipe_out[ EXECCMD_PIPE_WRITE ], &sa, 0 ) )
- {
- reportWindowsError( "CreatePipe" );
- exit( EXITBAD );
- }
- if ( globs.pipe_action && !CreatePipe( &cmdtab[ slot ].pipe_err[
- EXECCMD_PIPE_READ ], &cmdtab[ slot ].pipe_err[ EXECCMD_PIPE_WRITE ],
- &sa, 0 ) )
- {
- reportWindowsError( "CreatePipe" );
- exit( EXITBAD );
- }
-
- /* Set handle inheritance off for the pipe ends the parent reads from. */
- SetHandleInformation( cmdtab[ slot ].pipe_out[ EXECCMD_PIPE_READ ],
- HANDLE_FLAG_INHERIT, 0 );
- if ( globs.pipe_action )
- SetHandleInformation( cmdtab[ slot ].pipe_err[ EXECCMD_PIPE_READ ],
- HANDLE_FLAG_INHERIT, 0 );
-
- /* Hide the child window, if any. */
- si.dwFlags |= STARTF_USESHOWWINDOW;
- si.wShowWindow = SW_HIDE;
-
- /* Redirect the child's output streams to our pipes. */
- si.dwFlags |= STARTF_USESTDHANDLES;
- si.hStdOutput = cmdtab[ slot ].pipe_out[ EXECCMD_PIPE_WRITE ];
- si.hStdError = globs.pipe_action
- ? cmdtab[ slot ].pipe_err[ EXECCMD_PIPE_WRITE ]
- : cmdtab[ slot ].pipe_out[ EXECCMD_PIPE_WRITE ];
-
- /* Let the child inherit stdin, as some commands assume it is available. */
- si.hStdInput = GetStdHandle( STD_INPUT_HANDLE );
-
- /* Create output buffers. */
- string_new( cmdtab[ slot ].buffer_out );
- string_new( cmdtab[ slot ].buffer_err );
-
- if ( DEBUG_EXECCMD )
- printf( "Command string for CreateProcessA(): '%s'\n", command );
-
- /* Run the command by creating a sub-process for it. */
- if ( !CreateProcessA(
- NULL , /* application name */
- (char *)command , /* command line */
- NULL , /* process attributes */
- NULL , /* thread attributes */
- TRUE , /* inherit handles */
- CREATE_NEW_PROCESS_GROUP, /* create flags */
- NULL , /* env vars, null inherits env */
- NULL , /* current dir, null is our current dir */
- &si , /* startup info */
- &cmdtab[ slot ].pi ) ) /* child process info, if created */
- {
- reportWindowsError( "CreateProcessA" );
- exit( EXITBAD );
- }
-}
-
-
-/*
- * For more details on Windows cmd.exe shell command-line length limitations see
- * the following MSDN article:
- * http://support.microsoft.com/default.aspx?scid=kb;en-us;830473
- */
-
-static int raw_maxline()
-{
- OSVERSIONINFO os_info;
- os_info.dwOSVersionInfoSize = sizeof( os_info );
- GetVersionEx( &os_info );
-
- if ( os_info.dwMajorVersion >= 5 ) return 8191; /* XP */
- if ( os_info.dwMajorVersion == 4 ) return 2047; /* NT 4.x */
- return 996; /* NT 3.5.1 */
-}
-
-static int maxline()
-{
- static result;
- if ( !result ) result = raw_maxline();
- return result;
-}
-
-
-/*
- * Closes a Windows HANDLE and resets its variable to 0.
- */
-
-static void closeWinHandle( HANDLE * const handle )
-{
- if ( *handle )
- {
- CloseHandle( *handle );
- *handle = 0;
- }
-}
-
-
-/*
- * Frees and renews the given string.
- */
-
-static void string_renew( string * const s )
-{
- string_free( s );
- string_new( s );
-}
-
-
-/*
- * raw_command_length() - valid raw command string length
- *
- * Checks whether the given command may be executed as a raw command. If yes,
- * returns the corresponding command string length. If not, returns -1.
- *
- * Rules for constructing raw command strings:
- * - Command may not contain unquoted shell I/O redirection characters.
- * - May have at most one command line with non-whitespace content.
- * - Leading whitespace trimmed.
- * - Trailing all-whitespace lines trimmed.
- * - Trailing whitespace on the sole command line kept (may theoretically
- * affect the executed command).
- */
-
-static long raw_command_length( char const * command )
-{
- char const * p;
- char const * escape = 0;
- char inquote = 0;
- char const * newline = 0;
-
- /* Skip leading whitespace. */
- while ( isspace( *command ) )
- ++command;
-
- p = command;
-
- /* Look for newlines and unquoted I/O redirection. */
- do
- {
- p += strcspn( p, "\n\"'<>|\\" );
- switch ( *p )
- {
- case '\n':
- /* If our command contains non-whitespace content split over
- * multiple lines we can not execute it directly.
- */
- newline = p;
- while ( isspace( *++p ) );
- if ( *p ) return -1;
- break;
-
- case '\\':
- escape = escape && escape == p - 1 ? 0 : p;
- ++p;
- break;
-
- case '"':
- case '\'':
- if ( escape && escape == p - 1 )
- escape = 0;
- else if ( inquote == *p )
- inquote = 0;
- else if ( !inquote )
- inquote = *p;
- ++p;
- break;
-
- case '<':
- case '>':
- case '|':
- if ( !inquote )
- return -1;
- ++p;
- break;
- }
- }
- while ( *p );
-
- /* Return the number of characters the command will occupy. */
- return ( newline ? newline : p ) - command;
-}
-
-
-/* 64-bit arithmetic helpers. */
-
-/* Compute the carry bit from the addition of two 32-bit unsigned numbers. */
-#define add_carry_bit( a, b ) ((((a) | (b)) >> 31) & (~((a) + (b)) >> 31) & 0x1)
-
-/* Compute the high 32 bits of the addition of two 64-bit unsigned numbers, h1l1
- * and h2l2.
- */
-#define add_64_hi( h1, l1, h2, l2 ) ((h1) + (h2) + add_carry_bit(l1, l2))
-
-
-/*
- * Add two 64-bit unsigned numbers, h1l1 and h2l2.
- */
-
-static FILETIME add_64
-(
- unsigned long h1, unsigned long l1,
- unsigned long h2, unsigned long l2
-)
-{
- FILETIME result;
- result.dwLowDateTime = l1 + l2;
- result.dwHighDateTime = add_64_hi( h1, l1, h2, l2 );
- return result;
-}
-
-
-static FILETIME add_FILETIME( FILETIME t1, FILETIME t2 )
-{
- return add_64( t1.dwHighDateTime, t1.dwLowDateTime, t2.dwHighDateTime,
- t2.dwLowDateTime );
-}
-
-
-static FILETIME negate_FILETIME( FILETIME t )
-{
- /* 2s complement negation */
- return add_64( ~t.dwHighDateTime, ~t.dwLowDateTime, 0, 1 );
-}
-
-
-/*
- * filetime_to_seconds() - Windows FILETIME --> number of seconds conversion
- */
-
-static double filetime_to_seconds( FILETIME const ft )
-{
- return ft.dwHighDateTime * ( (double)( 1UL << 31 ) * 2.0 * 1.0e-7 ) +
- ft.dwLowDateTime * 1.0e-7;
-}
-
-
-static void record_times( HANDLE const process, timing_info * const time )
-{
- FILETIME creation;
- FILETIME exit;
- FILETIME kernel;
- FILETIME user;
- if ( GetProcessTimes( process, &creation, &exit, &kernel, &user ) )
- {
- time->system = filetime_to_seconds( kernel );
- time->user = filetime_to_seconds( user );
- timestamp_from_filetime( &time->start, &creation );
- timestamp_from_filetime( &time->end, &exit );
- }
-}
-
-
-#define IO_BUFFER_SIZE ( 16 * 1024 )
-
-static char ioBuffer[ IO_BUFFER_SIZE + 1 ];
-
-
-static void read_pipe
-(
- HANDLE in, /* the pipe to read from */
- string * out
-)
-{
- DWORD bytesInBuffer = 0;
- DWORD bytesAvailable = 0;
-
- do
- {
- /* check if we have any data to read */
- if ( !PeekNamedPipe( in, ioBuffer, IO_BUFFER_SIZE, &bytesInBuffer,
- &bytesAvailable, NULL ) )
- bytesAvailable = 0;
-
- /* read in the available data */
- if ( bytesAvailable > 0 )
- {
- /* we only read in the available bytes, to avoid blocking */
- if ( ReadFile( in, ioBuffer, bytesAvailable <= IO_BUFFER_SIZE ?
- bytesAvailable : IO_BUFFER_SIZE, &bytesInBuffer, NULL ) )
- {
- if ( bytesInBuffer > 0 )
- {
- /* Clean up some illegal chars. */
- int i;
- for ( i = 0; i < bytesInBuffer; ++i )
- {
- if ( ( (unsigned char)ioBuffer[ i ] < 1 ) )
- ioBuffer[ i ] = '?';
- }
- /* Null, terminate. */
- ioBuffer[ bytesInBuffer ] = '\0';
- /* Append to the output. */
- string_append( out, ioBuffer );
- /* Subtract what we read in. */
- bytesAvailable -= bytesInBuffer;
- }
- else
- {
- /* Likely read a error, bail out. */
- bytesAvailable = 0;
- }
- }
- else
- {
- /* Definitely read a error, bail out. */
- bytesAvailable = 0;
- }
- }
- }
- while ( bytesAvailable > 0 );
-}
-
-
-static void read_output()
-{
- int i;
- for ( i = 0; i < globs.jobs; ++i )
- if ( cmdtab[ i ].pi.hProcess )
- {
- /* Read stdout data. */
- if ( cmdtab[ i ].pipe_out[ EXECCMD_PIPE_READ ] )
- read_pipe( cmdtab[ i ].pipe_out[ EXECCMD_PIPE_READ ],
- cmdtab[ i ].buffer_out );
- /* Read stderr data. */
- if ( cmdtab[ i ].pipe_err[ EXECCMD_PIPE_READ ] )
- read_pipe( cmdtab[ i ].pipe_err[ EXECCMD_PIPE_READ ],
- cmdtab[ i ].buffer_err );
- }
-}
-
-
-/*
- * Waits for a single child process command to complete, or the timeout,
- * whichever comes first. Returns the index of the completed command in the
- * cmdtab array, or -1.
- */
-
-static int try_wait( int const timeoutMillis )
-{
- int i;
- int num_active;
- int wait_api_result;
- HANDLE active_handles[ MAXJOBS ];
- int active_procs[ MAXJOBS ];
-
- /* Prepare a list of all active processes to wait for. */
- for ( num_active = 0, i = 0; i < globs.jobs; ++i )
- if ( cmdtab[ i ].pi.hProcess )
- {
- active_handles[ num_active ] = cmdtab[ i ].pi.hProcess;
- active_procs[ num_active ] = i;
- ++num_active;
- }
-
- /* Wait for a child to complete, or for our timeout window to expire. */
- wait_api_result = WaitForMultipleObjects( num_active, active_handles,
- FALSE, timeoutMillis );
- if ( ( WAIT_OBJECT_0 <= wait_api_result ) &&
- ( wait_api_result < WAIT_OBJECT_0 + num_active ) )
- {
- /* Terminated process detected - return its index. */
- return active_procs[ wait_api_result - WAIT_OBJECT_0 ];
- }
-
- /* Timeout. */
- return -1;
-}
-
-
-static int try_kill_one()
-{
- /* Only need to check if a timeout was specified with the -l option. */
- if ( globs.timeout > 0 )
- {
- int i;
- for ( i = 0; i < globs.jobs; ++i )
- if ( cmdtab[ i ].pi.hProcess )
- {
- double const t = running_time( cmdtab[ i ].pi.hProcess );
- if ( t > (double)globs.timeout )
- {
- /* The job may have left an alert dialog around, try and get
- * rid of it before killing the job itself.
- */
- close_alert( &cmdtab[ i ].pi );
- /* We have a "runaway" job, kill it. */
- kill_process_tree( cmdtab[ i ].pi.dwProcessId,
- cmdtab[ i ].pi.hProcess );
- /* And return its running commands table slot. */
- return i;
- }
- }
- }
- return -1;
-}
-
-
-static void close_alerts()
-{
- /* We only attempt this every 5 seconds or so, because it is not a cheap
- * operation, and we will catch the alerts eventually. This check uses
- * floats as some compilers define CLOCKS_PER_SEC as a float or double.
- */
- if ( ( (float)clock() / (float)( CLOCKS_PER_SEC * 5 ) ) < ( 1.0 / 5.0 ) )
- {
- int i;
- for ( i = 0; i < globs.jobs; ++i )
- if ( cmdtab[ i ].pi.hProcess )
- close_alert( &cmdtab[ i ].pi );
- }
-}
-
-
-/*
- * Calc the current running time of an *active* process.
- */
-
-static double running_time( HANDLE const process )
-{
- FILETIME creation;
- FILETIME exit;
- FILETIME kernel;
- FILETIME user;
- if ( GetProcessTimes( process, &creation, &exit, &kernel, &user ) )
- {
- /* Compute the elapsed time. */
- FILETIME current;
- GetSystemTimeAsFileTime( ¤t );
- return filetime_to_seconds( add_FILETIME( current,
- negate_FILETIME( creation ) ) );
- }
- return 0.0;
-}
-
-
-/*
- * Not really optimal, or efficient, but it is easier this way, and it is not
- * like we are going to be killing thousands, or even tens of processes.
- */
-
-static void kill_process_tree( DWORD const pid, HANDLE const process )
-{
- HANDLE const process_snapshot_h = CreateToolhelp32Snapshot(
- TH32CS_SNAPPROCESS, 0 );
- if ( INVALID_HANDLE_VALUE != process_snapshot_h )
- {
- BOOL ok = TRUE;
- PROCESSENTRY32 pinfo;
- pinfo.dwSize = sizeof( PROCESSENTRY32 );
- for (
- ok = Process32First( process_snapshot_h, &pinfo );
- ok == TRUE;
- ok = Process32Next( process_snapshot_h, &pinfo ) )
- {
- if ( pinfo.th32ParentProcessID == pid )
- {
- /* Found a child, recurse to kill it and anything else below it.
- */
- HANDLE const ph = OpenProcess( PROCESS_ALL_ACCESS, FALSE,
- pinfo.th32ProcessID );
- if ( ph )
- {
- kill_process_tree( pinfo.th32ProcessID, ph );
- CloseHandle( ph );
- }
- }
- }
- CloseHandle( process_snapshot_h );
- }
- /* Now that the children are all dead, kill the root. */
- TerminateProcess( process, -2 );
-}
-
-
-static double creation_time( HANDLE const process )
-{
- FILETIME creation;
- FILETIME exit;
- FILETIME kernel;
- FILETIME user;
- return GetProcessTimes( process, &creation, &exit, &kernel, &user )
- ? filetime_to_seconds( creation )
- : 0.0;
-}
-
-
-/*
- * Recursive check if first process is parent (directly or indirectly) of the
- * second one. Both processes are passed as process ids, not handles. Special
- * return value 2 means that the second process is smss.exe and its parent
- * process is System (first argument is ignored).
- */
-
-static int is_parent_child( DWORD const parent, DWORD const child )
-{
- HANDLE process_snapshot_h = INVALID_HANDLE_VALUE;
-
- if ( !child )
- return 0;
- if ( parent == child )
- return 1;
-
- process_snapshot_h = CreateToolhelp32Snapshot( TH32CS_SNAPPROCESS, 0 );
- if ( INVALID_HANDLE_VALUE != process_snapshot_h )
- {
- BOOL ok = TRUE;
- PROCESSENTRY32 pinfo;
- pinfo.dwSize = sizeof( PROCESSENTRY32 );
- for (
- ok = Process32First( process_snapshot_h, &pinfo );
- ok == TRUE;
- ok = Process32Next( process_snapshot_h, &pinfo ) )
- {
- if ( pinfo.th32ProcessID == child )
- {
- /* Unfortunately, process ids are not really unique. There might
- * be spurious "parent and child" relationship match between two
- * non-related processes if real parent process of a given
- * process has exited (while child process kept running as an
- * "orphan") and the process id of such parent process has been
- * reused by internals of the operating system when creating
- * another process.
- *
- * Thus an additional check is needed - process creation time.
- * This check may fail (i.e. return 0) for system processes due
- * to insufficient privileges, and that is OK.
- */
- double tchild = 0.0;
- double tparent = 0.0;
- HANDLE const hchild = OpenProcess( PROCESS_QUERY_INFORMATION,
- FALSE, pinfo.th32ProcessID );
- CloseHandle( process_snapshot_h );
-
- /* csrss.exe may display message box like following:
- * xyz.exe - Unable To Locate Component
- * This application has failed to start because
- * boost_foo-bar.dll was not found. Re-installing the
- * application may fix the problem
- * This actually happens when starting a test process that
- * depends on a dynamic library which failed to build. We want
- * to automatically close these message boxes even though
- * csrss.exe is not our child process. We may depend on the fact
- * that (in all current versions of Windows) csrss.exe is a
- * direct child of the smss.exe process, which in turn is a
- * direct child of the System process, which always has process
- * id == 4. This check must be performed before comparing
- * process creation times.
- */
- if ( !stricmp( pinfo.szExeFile, "csrss.exe" ) &&
- is_parent_child( parent, pinfo.th32ParentProcessID ) == 2 )
- return 1;
- if ( !stricmp( pinfo.szExeFile, "smss.exe" ) &&
- ( pinfo.th32ParentProcessID == 4 ) )
- return 2;
-
- if ( hchild )
- {
- HANDLE hparent = OpenProcess( PROCESS_QUERY_INFORMATION,
- FALSE, pinfo.th32ParentProcessID );
- if ( hparent )
- {
- tchild = creation_time( hchild );
- tparent = creation_time( hparent );
- CloseHandle( hparent );
- }
- CloseHandle( hchild );
- }
-
- /* Return 0 if one of the following is true:
- * 1. we failed to read process creation time
- * 2. child was created before alleged parent
- */
- if ( ( tchild == 0.0 ) || ( tparent == 0.0 ) ||
- ( tchild < tparent ) )
- return 0;
-
- return is_parent_child( parent, pinfo.th32ParentProcessID ) & 1;
- }
- }
-
- CloseHandle( process_snapshot_h );
- }
-
- return 0;
-}
-
-
-/*
- * Called by the OS for each topmost window.
- */
-
-BOOL CALLBACK close_alert_window_enum( HWND hwnd, LPARAM lParam )
-{
- char buf[ 7 ] = { 0 };
- PROCESS_INFORMATION const * const pi = (PROCESS_INFORMATION *)lParam;
- DWORD pid;
- DWORD tid;
-
- /* We want to find and close any window that:
- * 1. is visible and
- * 2. is a dialog and
- * 3. is displayed by any of our child processes
- */
- if (
- /* We assume hidden windows do not require user interaction. */
- !IsWindowVisible( hwnd )
- /* Failed to read class name; presume it is not a dialog. */
- || !GetClassNameA( hwnd, buf, sizeof( buf ) )
- /* All Windows system dialogs use the same Window class name. */
- || strcmp( buf, "#32770" ) )
- return TRUE;
-
- /* GetWindowThreadProcessId() returns 0 on error, otherwise thread id of
- * the window's message pump thread.
- */
- tid = GetWindowThreadProcessId( hwnd, &pid );
- if ( !tid || !is_parent_child( pi->dwProcessId, pid ) )
- return TRUE;
-
- /* Ask real nice. */
- PostMessageA( hwnd, WM_CLOSE, 0, 0 );
-
- /* Wait and see if it worked. If not, insist. */
- if ( WaitForSingleObject( pi->hProcess, 200 ) == WAIT_TIMEOUT )
- {
- PostThreadMessageA( tid, WM_QUIT, 0, 0 );
- WaitForSingleObject( pi->hProcess, 300 );
- }
-
- /* Done, we do not want to check any other windows now. */
- return FALSE;
-}
-
-
-static void close_alert( PROCESS_INFORMATION const * const pi )
-{
- EnumWindows( &close_alert_window_enum, (LPARAM)pi );
-}
-
-
-/*
- * Open a command file to store the command into for executing using an external
- * shell. Returns a pointer to a FILE open for writing or 0 in case such a file
- * could not be opened. The file name used is stored back in the corresponding
- * running commands table slot.
- *
- * Expects the running commands table slot's command_file attribute to contain
- * either a zeroed out string object or one prepared previously by this same
- * function.
- */
-
-static FILE * open_command_file( int const slot )
-{
- string * const command_file = cmdtab[ slot ].command_file;
-
- /* If the temporary command file name has not already been prepared for this
- * slot number, prepare a new one containing a '##' place holder that will
- * be changed later and needs to be located at a fixed distance from the
- * end.
- */
- if ( !command_file->value )
- {
- DWORD const procID = GetCurrentProcessId();
- string const * const tmpdir = path_tmpdir();
- string_new( command_file );
- string_reserve( command_file, tmpdir->size + 64 );
- command_file->size = sprintf( command_file->value,
- "%s\\jam%d-%02d-##.bat", tmpdir->value, procID, slot );
- }
-
- /* For some reason opening a command file can fail intermittently. But doing
- * some retries works. Most likely this is due to a previously existing file
- * of the same name that happens to still be opened by an active virus
- * scanner. Originally pointed out and fixed by Bronek Kozicki.
- *
- * We first try to open several differently named files to avoid having to
- * wait idly if not absolutely necessary. Our temporary command file names
- * contain a fixed position place holder we use for generating different
- * file names.
- */
- {
- char * const index1 = command_file->value + command_file->size - 6;
- char * const index2 = index1 + 1;
- int waits_remaining;
- assert( command_file->value < index1 );
- assert( index2 + 1 < command_file->value + command_file->size );
- assert( index2[ 1 ] == '.' );
- for ( waits_remaining = 3; ; --waits_remaining )
- {
- int index;
- for ( index = 0; index != 20; ++index )
- {
- FILE * f;
- *index1 = '0' + index / 10;
- *index2 = '0' + index % 10;
- f = fopen( command_file->value, "w" );
- if ( f ) return f;
- }
- if ( !waits_remaining ) break;
- Sleep( 250 );
- }
- }
-
- return 0;
-}
-
-
-/*
- * Prepare a command file to be executed using an external shell.
- */
-
-static char const * prepare_command_file( string const * command, int slot )
-{
- FILE * const f = open_command_file( slot );
- if ( !f )
- {
- printf( "failed to write command file!\n" );
- exit( EXITBAD );
- }
- fputs( command->value, f );
- fclose( f );
- return cmdtab[ slot ].command_file->value;
-}
-
-
-/*
- * Find a free slot in the running commands table.
- */
-
-static int get_free_cmdtab_slot()
-{
- int slot;
- for ( slot = 0; slot < MAXJOBS; ++slot )
- if ( !cmdtab[ slot ].pi.hProcess )
- return slot;
- printf( "no slots for child!\n" );
- exit( EXITBAD );
-}
-
-
-/*
- * Put together the final command string we are to run.
- */
-
-static void string_new_from_argv( string * result, char const * const * argv )
-{
- assert( argv );
- assert( argv[ 0 ] );
- string_copy( result, *(argv++) );
- while ( *argv )
- {
- string_push_back( result, ' ' );
- string_append( result, *(argv++) );
- }
-}
-
-
-/*
- * Reports the last failed Windows API related error message.
- */
-
-static void reportWindowsError( char const * const apiName )
-{
- char * errorMessage;
- DWORD const errorCode = GetLastError();
- DWORD apiResult = FormatMessageA(
- FORMAT_MESSAGE_ALLOCATE_BUFFER | /* __in DWORD dwFlags */
- FORMAT_MESSAGE_FROM_SYSTEM |
- FORMAT_MESSAGE_IGNORE_INSERTS,
- NULL, /* __in_opt LPCVOID lpSource */
- errorCode, /* __in DWORD dwMessageId */
- 0, /* __in DWORD dwLanguageId */
- (LPSTR)&errorMessage, /* __out LPTSTR lpBuffer */
- 0, /* __in DWORD nSize */
- 0 ); /* __in_opt va_list * Arguments */
- if ( !apiResult )
- printf( "%s() Windows API failed: %d.\n", apiName, errorCode );
- else
- {
- printf( "%s() Windows API failed: %d - %s\n", apiName, errorCode,
- errorMessage );
- LocalFree( errorMessage );
- }
-}
-
-
-#endif /* USE_EXECNT */
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/execunix.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/execunix.c b/ext/kenlm/jam-files/engine/execunix.c
deleted file mode 100644
index 965e580..0000000
--- a/ext/kenlm/jam-files/engine/execunix.c
+++ /dev/null
@@ -1,559 +0,0 @@
-/*
- * Copyright 1993, 1995 Christopher Seiwald.
- * Copyright 2007 Noel Belcourt.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-#include "jam.h"
-#include "execcmd.h"
-
-#include "lists.h"
-#include "output.h"
-#include "strings.h"
-
-#include <errno.h>
-#include <signal.h>
-#include <stdio.h>
-#include <time.h>
-#include <unistd.h> /* vfork(), _exit(), STDOUT_FILENO and such */
-#include <sys/resource.h>
-#include <sys/times.h>
-#include <sys/wait.h>
-
-#if defined(sun) || defined(__sun)
- #include <wait.h>
-#endif
-
-#ifdef USE_EXECUNIX
-
-#include <sys/times.h>
-
-#if defined(__APPLE__)
- #define NO_VFORK
-#endif
-
-#ifdef NO_VFORK
- #define vfork() fork()
-#endif
-
-
-/*
- * execunix.c - execute a shell script on UNIX/OS2/AmigaOS
- *
- * If $(JAMSHELL) is defined, uses that to formulate execvp()/spawnvp(). The
- * default is: /bin/sh -c
- *
- * In $(JAMSHELL), % expands to the command string and ! expands to the slot
- * number (starting at 1) for multiprocess (-j) invocations. If $(JAMSHELL) does
- * not include a %, it is tacked on as the last argument.
- *
- * Each word must be an individual element in a jam variable value.
- *
- * Do not just set JAMSHELL to /bin/sh - it will not work!
- *
- * External routines:
- * exec_check() - preprocess and validate the command.
- * exec_cmd() - launch an async command execution.
- * exec_wait() - wait for any of the async command processes to terminate.
- */
-
-/* find a free slot in the running commands table */
-static int get_free_cmdtab_slot();
-
-/* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */
-
-static clock_t tps;
-static int old_time_initialized;
-static struct tms old_time;
-
-/* We hold stdout & stderr child process information in two element arrays
- * indexed as follows.
- */
-#define OUT 0
-#define ERR 1
-
-static struct
-{
- int pid; /* on win32, a real process handle */
- int fd[ 2 ]; /* file descriptors for stdout and stderr */
- FILE * stream[ 2 ]; /* child's stdout and stderr file streams */
- clock_t start_time; /* start time of child process */
- int exit_reason; /* termination status */
- char * buffer[ 2 ]; /* buffers to hold stdout and stderr, if any */
- int buf_size[ 2 ]; /* buffer sizes in bytes */
- timestamp start_dt; /* start of command timestamp */
-
- /* Function called when the command completes. */
- ExecCmdCallback func;
-
- /* Opaque data passed back to the 'func' callback. */
- void * closure;
-} cmdtab[ MAXJOBS ] = { { 0 } };
-
-
-/*
- * exec_check() - preprocess and validate the command.
- */
-
-int exec_check
-(
- string const * command,
- LIST * * pShell,
- int * error_length,
- int * error_max_length
-)
-{
- int const is_raw_cmd = is_raw_command_request( *pShell );
-
- /* We allow empty commands for non-default shells since we do not really
- * know what they are going to do with such commands.
- */
- if ( !command->size && ( is_raw_cmd || list_empty( *pShell ) ) )
- return EXEC_CHECK_NOOP;
-
- return is_raw_cmd
- ? EXEC_CHECK_OK
- : check_cmd_for_too_long_lines( command->value, MAXLINE, error_length,
- error_max_length );
-}
-
-
-/*
- * exec_cmd() - launch an async command execution.
- */
-
-/* We hold file descriptors for pipes used to communicate with child processes
- * in two element arrays indexed as follows.
- */
-#define EXECCMD_PIPE_READ 0
-#define EXECCMD_PIPE_WRITE 1
-
-void exec_cmd
-(
- string const * command,
- ExecCmdCallback func,
- void * closure,
- LIST * shell
-)
-{
- int const slot = get_free_cmdtab_slot();
- int out[ 2 ];
- int err[ 2 ];
- int len;
- char const * argv[ MAXARGC + 1 ]; /* +1 for NULL */
-
- /* Initialize default shell. */
- static LIST * default_shell;
- if ( !default_shell )
- default_shell = list_push_back( list_new(
- object_new( "/bin/sh" ) ),
- object_new( "-c" ) );
-
- if ( list_empty( shell ) )
- shell = default_shell;
-
- /* Forumulate argv. If shell was defined, be prepared for % and ! subs.
- * Otherwise, use stock /bin/sh.
- */
- argv_from_shell( argv, shell, command->value, slot );
-
- if ( DEBUG_EXECCMD )
- {
- int i;
- printf( "Using shell: " );
- list_print( shell );
- printf( "\n" );
- for ( i = 0; argv[ i ]; ++i )
- printf( " argv[%d] = '%s'\n", i, argv[ i ] );
- }
-
- /* Create pipes for collecting child output. */
- if ( pipe( out ) < 0 || ( globs.pipe_action && pipe( err ) < 0 ) )
- {
- perror( "pipe" );
- exit( EXITBAD );
- }
-
- /* Initialize old_time only once. */
- if ( !old_time_initialized )
- {
- times( &old_time );
- old_time_initialized = 1;
- }
-
- /* Start the command */
-
- timestamp_current( &cmdtab[ slot ].start_dt );
-
- if ( 0 < globs.timeout )
- {
- /* Handle hung processes by manually tracking elapsed time and signal
- * process when time limit expires.
- */
- struct tms buf;
- cmdtab[ slot ].start_time = times( &buf );
-
- /* Make a global, only do this once. */
- if ( !tps ) tps = sysconf( _SC_CLK_TCK );
- }
-
- /* Child does not need the read pipe ends used by the parent. */
- fcntl( out[ EXECCMD_PIPE_READ ], F_SETFD, FD_CLOEXEC );
- if ( globs.pipe_action )
- fcntl( err[ EXECCMD_PIPE_READ ], F_SETFD, FD_CLOEXEC );
-
- if ( ( cmdtab[ slot ].pid = vfork() ) == -1 )
- {
- perror( "vfork" );
- exit( EXITBAD );
- }
-
- if ( cmdtab[ slot ].pid == 0 )
- {
- /*****************/
- /* Child process */
- /*****************/
- int const pid = getpid();
-
- /* Redirect stdout and stderr to pipes inherited from the parent. */
- dup2( out[ EXECCMD_PIPE_WRITE ], STDOUT_FILENO );
- dup2( globs.pipe_action ? err[ EXECCMD_PIPE_WRITE ] :
- out[ EXECCMD_PIPE_WRITE ], STDERR_FILENO );
- close( out[ EXECCMD_PIPE_WRITE ] );
- if ( globs.pipe_action )
- close( err[ EXECCMD_PIPE_WRITE ] );
-
- /* Make this process a process group leader so that when we kill it, all
- * child processes of this process are terminated as well. We use
- * killpg( pid, SIGKILL ) to kill the process group leader and all its
- * children.
- */
- if ( 0 < globs.timeout )
- {
- struct rlimit r_limit;
- r_limit.rlim_cur = globs.timeout;
- r_limit.rlim_max = globs.timeout;
- setrlimit( RLIMIT_CPU, &r_limit );
- }
- setpgid( pid, pid );
- execvp( argv[ 0 ], (char * *)argv );
- perror( "execvp" );
- _exit( 127 );
- }
-
- /******************/
- /* Parent process */
- /******************/
- setpgid( cmdtab[ slot ].pid, cmdtab[ slot ].pid );
-
- /* Parent not need the write pipe ends used by the child. */
- close( out[ EXECCMD_PIPE_WRITE ] );
- if ( globs.pipe_action )
- close( err[ EXECCMD_PIPE_WRITE ] );
-
- /* Set both pipe read file descriptors to non-blocking. */
- fcntl( out[ EXECCMD_PIPE_READ ], F_SETFL, O_NONBLOCK );
- if ( globs.pipe_action )
- fcntl( err[ EXECCMD_PIPE_READ ], F_SETFL, O_NONBLOCK );
-
- /* Parent reads from out[ EXECCMD_PIPE_READ ]. */
- cmdtab[ slot ].fd[ OUT ] = out[ EXECCMD_PIPE_READ ];
- cmdtab[ slot ].stream[ OUT ] = fdopen( cmdtab[ slot ].fd[ OUT ], "rb" );
- if ( !cmdtab[ slot ].stream[ OUT ] )
- {
- perror( "fdopen" );
- exit( EXITBAD );
- }
-
- /* Parent reads from err[ EXECCMD_PIPE_READ ]. */
- if ( globs.pipe_action )
- {
- cmdtab[ slot ].fd[ ERR ] = err[ EXECCMD_PIPE_READ ];
- cmdtab[ slot ].stream[ ERR ] = fdopen( cmdtab[ slot ].fd[ ERR ], "rb" );
- if ( !cmdtab[ slot ].stream[ ERR ] )
- {
- perror( "fdopen" );
- exit( EXITBAD );
- }
- }
-
- /* Save input data into the selected running commands table slot. */
- cmdtab[ slot ].func = func;
- cmdtab[ slot ].closure = closure;
-}
-
-#undef EXECCMD_PIPE_READ
-#undef EXECCMD_PIPE_WRITE
-
-
-/* Returns 1 if file descriptor is closed, or 0 if it is still alive.
- *
- * i is index into cmdtab
- *
- * s (stream) indexes:
- * - cmdtab[ i ].stream[ s ]
- * - cmdtab[ i ].buffer[ s ]
- * - cmdtab[ i ].fd [ s ]
- */
-
-static int read_descriptor( int i, int s )
-{
- int ret;
- char buffer[ BUFSIZ ];
-
- while ( 0 < ( ret = fread( buffer, sizeof( char ), BUFSIZ - 1,
- cmdtab[ i ].stream[ s ] ) ) )
- {
- buffer[ ret ] = 0;
- if ( !cmdtab[ i ].buffer[ s ] )
- {
- /* Never been allocated. */
- if ( globs.max_buf && ret > globs.max_buf )
- {
- ret = globs.max_buf;
- buffer[ ret ] = 0;
- }
- cmdtab[ i ].buf_size[ s ] = ret + 1;
- cmdtab[ i ].buffer[ s ] = (char*)BJAM_MALLOC_ATOMIC( ret + 1 );
- memcpy( cmdtab[ i ].buffer[ s ], buffer, ret + 1 );
- }
- else
- {
- /* Previously allocated. */
- if ( cmdtab[ i ].buf_size[ s ] < globs.max_buf || !globs.max_buf )
- {
- char * tmp = cmdtab[ i ].buffer[ s ];
- int const old_len = cmdtab[ i ].buf_size[ s ] - 1;
- int const new_len = old_len + ret + 1;
- cmdtab[ i ].buf_size[ s ] = new_len;
- cmdtab[ i ].buffer[ s ] = (char*)BJAM_MALLOC_ATOMIC( new_len );
- memcpy( cmdtab[ i ].buffer[ s ], tmp, old_len );
- memcpy( cmdtab[ i ].buffer[ s ] + old_len, buffer, ret + 1 );
- BJAM_FREE( tmp );
- }
- }
- }
-
- /* If buffer full, ensure last buffer char is newline so that jam log
- * contains the command status at beginning of it own line instead of
- * appended to end of the previous output.
- */
- if ( globs.max_buf && globs.max_buf <= cmdtab[ i ].buf_size[ s ] )
- cmdtab[ i ].buffer[ s ][ cmdtab[ i ].buf_size[ s ] - 2 ] = '\n';
-
- return feof( cmdtab[ i ].stream[ s ] );
-}
-
-
-/*
- * close_streams() - Close the stream and pipe descriptor.
- */
-
-static void close_streams( int const i, int const s )
-{
- fclose( cmdtab[ i ].stream[ s ] );
- cmdtab[ i ].stream[ s ] = 0;
-
- close( cmdtab[ i ].fd[ s ] );
- cmdtab[ i ].fd[ s ] = 0;
-}
-
-
-/*
- * Populate the file descriptors collection for use in select() and return the
- * maximal included file descriptor value.
- */
-
-static int populate_file_descriptors( fd_set * const fds )
-{
- int i;
- int fd_max = 0;
-
- FD_ZERO( fds );
- for ( i = 0; i < globs.jobs; ++i )
- {
- int fd;
- if ( ( fd = cmdtab[ i ].fd[ OUT ] ) > 0 )
- {
- if ( fd > fd_max ) fd_max = fd;
- FD_SET( fd, fds );
- }
- if ( globs.pipe_action )
- {
- if ( ( fd = cmdtab[ i ].fd[ ERR ] ) > 0 )
- {
- if ( fd > fd_max ) fd_max = fd;
- FD_SET( fd, fds );
- }
- }
- }
- return fd_max;
-}
-
-
-/*
- * exec_wait() - wait for any of the async command processes to terminate.
- *
- * May register more than one terminated child process but will exit as soon as
- * at least one has been registered.
- */
-
-void exec_wait()
-{
- int finished = 0;
-
- /* Process children that signaled. */
- while ( !finished )
- {
- int i;
- struct timeval tv;
- struct timeval * ptv = NULL;
- int select_timeout = globs.timeout;
-
- /* Prepare file descriptor information for use in select(). */
- fd_set fds;
- int const fd_max = populate_file_descriptors( &fds );
-
- /* Check for timeouts:
- * - kill children that already timed out
- * - decide how long until the next one times out
- */
- if ( globs.timeout > 0 )
- {
- struct tms buf;
- clock_t const current = times( &buf );
- for ( i = 0; i < globs.jobs; ++i )
- if ( cmdtab[ i ].pid )
- {
- clock_t const consumed =
- ( current - cmdtab[ i ].start_time ) / tps;
- if ( consumed >= globs.timeout )
- {
- killpg( cmdtab[ i ].pid, SIGKILL );
- cmdtab[ i ].exit_reason = EXIT_TIMEOUT;
- }
- else if ( globs.timeout - consumed < select_timeout )
- select_timeout = globs.timeout - consumed;
- }
-
- /* If nothing else causes our select() call to exit, force it after
- * however long it takes for the next one of our child processes to
- * crossed its alloted processing time so we can terminate it.
- */
- tv.tv_sec = select_timeout;
- tv.tv_usec = 0;
- ptv = &tv;
- }
-
- /* select() will wait for I/O on a descriptor, a signal, or timeout. */
- {
- int ret;
- while ( ( ret = select( fd_max + 1, &fds, 0, 0, ptv ) ) == -1 )
- if ( errno != EINTR )
- break;
- if ( ret <= 0 )
- continue;
- }
-
- for ( i = 0; i < globs.jobs; ++i )
- {
- int out_done = 0;
- int err_done = 0;
- if ( FD_ISSET( cmdtab[ i ].fd[ OUT ], &fds ) )
- out_done = read_descriptor( i, OUT );
-
- if ( globs.pipe_action && FD_ISSET( cmdtab[ i ].fd[ ERR ], &fds ) )
- err_done = read_descriptor( i, ERR );
-
- /* If feof on either descriptor, we are done. */
- if ( out_done || err_done )
- {
- int pid;
- int status;
- int rstat;
- timing_info time_info;
-
- /* We found a terminated child process - our search is done. */
- finished = 1;
-
- /* Close the stream and pipe descriptors. */
- close_streams( i, OUT );
- if ( globs.pipe_action )
- close_streams( i, ERR );
-
- /* Reap the child and release resources. */
- while ( ( pid = waitpid( cmdtab[ i ].pid, &status, 0 ) ) == -1 )
- if ( errno != EINTR )
- break;
- if ( pid != cmdtab[ i ].pid )
- {
- printf( "unknown pid %d with errno = %d\n", pid, errno );
- exit( EXITBAD );
- }
-
- /* Set reason for exit if not timed out. */
- if ( WIFEXITED( status ) )
- cmdtab[ i ].exit_reason = WEXITSTATUS( status )
- ? EXIT_FAIL
- : EXIT_OK;
-
- {
- struct tms new_time;
- times( &new_time );
- time_info.system = (double)( new_time.tms_cstime -
- old_time.tms_cstime ) / CLOCKS_PER_SEC;
- time_info.user = (double)( new_time.tms_cutime -
- old_time.tms_cutime ) / CLOCKS_PER_SEC;
- timestamp_copy( &time_info.start, &cmdtab[ i ].start_dt );
- timestamp_current( &time_info.end );
- old_time = new_time;
- }
-
- /* Drive the completion. */
- if ( interrupted() )
- rstat = EXEC_CMD_INTR;
- else if ( status )
- rstat = EXEC_CMD_FAIL;
- else
- rstat = EXEC_CMD_OK;
-
- /* Call the callback, may call back to jam rule land. */
- (*cmdtab[ i ].func)( cmdtab[ i ].closure, rstat, &time_info,
- cmdtab[ i ].buffer[ OUT ], cmdtab[ i ].buffer[ ERR ],
- cmdtab[ i ].exit_reason );
-
- /* Clean up the command's running commands table slot. */
- BJAM_FREE( cmdtab[ i ].buffer[ OUT ] );
- cmdtab[ i ].buffer[ OUT ] = 0;
- cmdtab[ i ].buf_size[ OUT ] = 0;
-
- BJAM_FREE( cmdtab[ i ].buffer[ ERR ] );
- cmdtab[ i ].buffer[ ERR ] = 0;
- cmdtab[ i ].buf_size[ ERR ] = 0;
-
- cmdtab[ i ].pid = 0;
- cmdtab[ i ].func = 0;
- cmdtab[ i ].closure = 0;
- cmdtab[ i ].start_time = 0;
- }
- }
- }
-}
-
-
-/*
- * Find a free slot in the running commands table.
- */
-
-static int get_free_cmdtab_slot()
-{
- int slot;
- for ( slot = 0; slot < MAXJOBS; ++slot )
- if ( !cmdtab[ slot ].pid )
- return slot;
- printf( "no slots for child!\n" );
- exit( EXITBAD );
-}
-
-# endif /* USE_EXECUNIX */
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/filent.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/filent.c b/ext/kenlm/jam-files/engine/filent.c
deleted file mode 100644
index e4ac331..0000000
--- a/ext/kenlm/jam-files/engine/filent.c
+++ /dev/null
@@ -1,435 +0,0 @@
-/*
- * Copyright 1993, 1995 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Copyright 2005 Rene Rivera.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * filent.c - scan directories and archives on NT
- *
- * External routines:
- * file_archscan() - scan an archive for files
- * file_mkdir() - create a directory
- * file_supported_fmt_resolution() - file modification timestamp resolution
- *
- * External routines called only via routines in filesys.c:
- * file_collect_dir_content_() - collects directory content information
- * file_dirscan_() - OS specific file_dirscan() implementation
- * file_query_() - query information about a path from the OS
- */
-
-#include "jam.h"
-#ifdef OS_NT
-#include "filesys.h"
-
-#include "object.h"
-#include "pathsys.h"
-#include "strings.h"
-
-#ifdef __BORLANDC__
-# undef FILENAME /* cpp namespace collision */
-#endif
-
-#define WIN32_LEAN_AND_MEAN
-#include <windows.h>
-
-#include <assert.h>
-#include <ctype.h>
-#include <direct.h>
-#include <io.h>
-
-
-/*
- * file_collect_dir_content_() - collects directory content information
- */
-
-int file_collect_dir_content_( file_info_t * const d )
-{
- PATHNAME f;
- string pathspec[ 1 ];
- string pathname[ 1 ];
- LIST * files = L0;
- int d_length;
-
- assert( d );
- assert( d->is_dir );
- assert( list_empty( d->files ) );
-
- d_length = strlen( object_str( d->name ) );
-
- memset( (char *)&f, '\0', sizeof( f ) );
- f.f_dir.ptr = object_str( d->name );
- f.f_dir.len = d_length;
-
- /* Prepare file search specification for the FindXXX() Windows API. */
- if ( !d_length )
- string_copy( pathspec, ".\\*" );
- else
- {
- /* We can not simply assume the given folder name will never include its
- * trailing path separator or otherwise we would not support the Windows
- * root folder specified without its drive letter, i.e. '\'.
- */
- char const trailingChar = object_str( d->name )[ d_length - 1 ] ;
- string_copy( pathspec, object_str( d->name ) );
- if ( ( trailingChar != '\\' ) && ( trailingChar != '/' ) )
- string_append( pathspec, "\\" );
- string_append( pathspec, "*" );
- }
-
- /* The following code for collecting information about all files in a folder
- * needs to be kept synchronized with how the file_query() operation is
- * implemented (collects information about a single file).
- */
- {
- /* FIXME: Avoid duplicate FindXXX Windows API calls here and in the code
- * determining a normalized path.
- */
- WIN32_FIND_DATA finfo;
- HANDLE const findHandle = FindFirstFileA( pathspec->value, &finfo );
- if ( findHandle == INVALID_HANDLE_VALUE )
- {
- string_free( pathspec );
- return -1;
- }
-
- string_new( pathname );
- do
- {
- OBJECT * pathname_obj;
-
- f.f_base.ptr = finfo.cFileName;
- f.f_base.len = strlen( finfo.cFileName );
-
- string_truncate( pathname, 0 );
- path_build( &f, pathname );
-
- pathname_obj = object_new( pathname->value );
- path_register_key( pathname_obj );
- files = list_push_back( files, pathname_obj );
- {
- int found;
- file_info_t * const ff = file_info( pathname_obj, &found );
- ff->is_dir = finfo.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY;
- ff->is_file = !ff->is_dir;
- ff->exists = 1;
- timestamp_from_filetime( &ff->time, &finfo.ftLastWriteTime );
- }
- }
- while ( FindNextFile( findHandle, &finfo ) );
-
- FindClose( findHandle );
- }
-
- string_free( pathname );
- string_free( pathspec );
-
- d->files = files;
- return 0;
-}
-
-
-/*
- * file_dirscan_() - OS specific file_dirscan() implementation
- */
-
-void file_dirscan_( file_info_t * const d, scanback func, void * closure )
-{
- assert( d );
- assert( d->is_dir );
-
- /* Special case \ or d:\ : enter it */
- {
- char const * const name = object_str( d->name );
- if ( name[ 0 ] == '\\' && !name[ 1 ] )
- {
- (*func)( closure, d->name, 1 /* stat()'ed */, &d->time );
- }
- else if ( name[ 0 ] && name[ 1 ] == ':' && name[ 2 ] && !name[ 3 ] )
- {
- /* We have just entered a 3-letter drive name spelling (with a
- * trailing slash), into the hash table. Now enter its two-letter
- * variant, without the trailing slash, so that if we try to check
- * whether "c:" exists, we hit it.
- *
- * Jam core has workarounds for that. Given:
- * x = c:\whatever\foo ;
- * p = $(x:D) ;
- * p2 = $(p:D) ;
- * There will be no trailing slash in $(p), but there will be one in
- * $(p2). But, that seems rather fragile.
- */
- OBJECT * const dir_no_slash = object_new_range( name, 2 );
- (*func)( closure, d->name, 1 /* stat()'ed */, &d->time );
- (*func)( closure, dir_no_slash, 1 /* stat()'ed */, &d->time );
- object_free( dir_no_slash );
- }
- }
-}
-
-
-/*
- * file_mkdir() - create a directory
- */
-
-int file_mkdir( char const * const path )
-{
- return _mkdir( path );
-}
-
-
-/*
- * file_query_() - query information about a path from the OS
- *
- * The following code for collecting information about a single file needs to be
- * kept synchronized with how the file_collect_dir_content_() operation is
- * implemented (collects information about all files in a folder).
- */
-
-int try_file_query_root( file_info_t * const info )
-{
- WIN32_FILE_ATTRIBUTE_DATA fileData;
- char buf[ 4 ];
- char const * const pathstr = object_str( info->name );
- if ( !pathstr[ 0 ] )
- {
- buf[ 0 ] = '.';
- buf[ 1 ] = 0;
- }
- else if ( pathstr[ 0 ] == '\\' && ! pathstr[ 1 ] )
- {
- buf[ 0 ] = '\\';
- buf[ 1 ] = '\0';
- }
- else if ( pathstr[ 1 ] == ':' )
- {
- if ( !pathstr[ 2 ] )
- {
- }
- else if ( !pathstr[ 2 ] || ( pathstr[ 2 ] == '\\' && !pathstr[ 3 ] ) )
- {
- buf[ 0 ] = pathstr[ 0 ];
- buf[ 1 ] = ':';
- buf[ 2 ] = '\\';
- buf[ 3 ] = '\0';
- }
- else
- {
- return 0;
- }
- }
- else
- {
- return 0;
- }
-
- /* We have a root path */
- if ( !GetFileAttributesExA( buf, GetFileExInfoStandard, &fileData ) )
- {
- info->is_dir = 0;
- info->is_file = 0;
- info->exists = 0;
- timestamp_clear( &info->time );
- }
- else
- {
- info->is_dir = fileData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY;
- info->is_file = !info->is_dir;
- info->exists = 1;
- timestamp_from_filetime( &info->time, &fileData.ftLastWriteTime );
- }
- return 1;
-}
-
-void file_query_( file_info_t * const info )
-{
- char const * const pathstr = object_str( info->name );
- const char * dir;
- OBJECT * parent;
- file_info_t * parent_info;
-
- if ( try_file_query_root( info ) )
- return;
-
- if ( ( dir = strrchr( pathstr, '\\' ) ) )
- {
- parent = object_new_range( pathstr, dir - pathstr );
- }
- else
- {
- parent = object_copy( constant_empty );
- }
- parent_info = file_query( parent );
- object_free( parent );
- if ( !parent_info || !parent_info->is_dir )
- {
- info->is_dir = 0;
- info->is_file = 0;
- info->exists = 0;
- timestamp_clear( &info->time );
- }
- else
- {
- info->is_dir = 0;
- info->is_file = 0;
- info->exists = 0;
- timestamp_clear( &info->time );
- if ( list_empty( parent_info->files ) )
- file_collect_dir_content_( parent_info );
- }
-}
-
-
-/*
- * file_supported_fmt_resolution() - file modification timestamp resolution
- *
- * Returns the minimum file modification timestamp resolution supported by this
- * Boost Jam implementation. File modification timestamp changes of less than
- * the returned value might not be recognized.
- *
- * Does not take into consideration any OS or file system related restrictions.
- *
- * Return value 0 indicates that any value supported by the OS is also supported
- * here.
- */
-
-void file_supported_fmt_resolution( timestamp * const t )
-{
- /* On Windows we support nano-second file modification timestamp resolution,
- * just the same as the Windows OS itself.
- */
- timestamp_init( t, 0, 0 );
-}
-
-
-/*
- * file_archscan() - scan an archive for files
- */
-
-/* Straight from SunOS */
-
-#define ARMAG "!<arch>\n"
-#define SARMAG 8
-
-#define ARFMAG "`\n"
-
-struct ar_hdr
-{
- char ar_name[ 16 ];
- char ar_date[ 12 ];
- char ar_uid[ 6 ];
- char ar_gid[ 6 ];
- char ar_mode[ 8 ];
- char ar_size[ 10 ];
- char ar_fmag[ 2 ];
-};
-
-#define SARFMAG 2
-#define SARHDR sizeof( struct ar_hdr )
-
-void file_archscan( char const * archive, scanback func, void * closure )
-{
- struct ar_hdr ar_hdr;
- char * string_table = 0;
- char buf[ MAXJPATH ];
- long offset;
- int const fd = open( archive, O_RDONLY | O_BINARY, 0 );
-
- if ( fd < 0 )
- return;
-
- if ( read( fd, buf, SARMAG ) != SARMAG || strncmp( ARMAG, buf, SARMAG ) )
- {
- close( fd );
- return;
- }
-
- offset = SARMAG;
-
- if ( DEBUG_BINDSCAN )
- printf( "scan archive %s\n", archive );
-
- while ( ( read( fd, &ar_hdr, SARHDR ) == SARHDR ) &&
- !memcmp( ar_hdr.ar_fmag, ARFMAG, SARFMAG ) )
- {
- long lar_date;
- long lar_size;
- char * name = 0;
- char * endname;
-
- sscanf( ar_hdr.ar_date, "%ld", &lar_date );
- sscanf( ar_hdr.ar_size, "%ld", &lar_size );
-
- lar_size = ( lar_size + 1 ) & ~1;
-
- if ( ar_hdr.ar_name[ 0 ] == '/' && ar_hdr.ar_name[ 1 ] == '/' )
- {
- /* This is the "string table" entry of the symbol table, holding
- * filename strings longer than 15 characters, i.e. those that do
- * not fit into ar_name.
- */
- string_table = BJAM_MALLOC_ATOMIC( lar_size + 1 );
- if ( read( fd, string_table, lar_size ) != lar_size )
- printf( "error reading string table\n" );
- string_table[ lar_size ] = '\0';
- offset += SARHDR + lar_size;
- continue;
- }
- else if ( ar_hdr.ar_name[ 0 ] == '/' && ar_hdr.ar_name[ 1 ] != ' ' )
- {
- /* Long filenames are recognized by "/nnnn" where nnnn is the
- * string's offset in the string table represented in ASCII
- * decimals.
- */
- name = string_table + atoi( ar_hdr.ar_name + 1 );
- for ( endname = name; *endname && *endname != '\n'; ++endname );
- }
- else
- {
- /* normal name */
- name = ar_hdr.ar_name;
- endname = name + sizeof( ar_hdr.ar_name );
- }
-
- /* strip trailing white-space, slashes, and backslashes */
-
- while ( endname-- > name )
- if ( !isspace( *endname ) && ( *endname != '\\' ) && ( *endname !=
- '/' ) )
- break;
- *++endname = 0;
-
- /* strip leading directory names, an NT specialty */
- {
- char * c;
- if ( c = strrchr( name, '/' ) )
- name = c + 1;
- if ( c = strrchr( name, '\\' ) )
- name = c + 1;
- }
-
- sprintf( buf, "%s(%.*s)", archive, endname - name, name );
- {
- OBJECT * const member = object_new( buf );
- timestamp time;
- timestamp_init( &time, (time_t)lar_date, 0 );
- (*func)( closure, member, 1 /* time valid */, &time );
- object_free( member );
- }
-
- offset += SARHDR + lar_size;
- lseek( fd, offset, 0 );
- }
-
- close( fd );
-}
-
-#endif /* OS_NT */
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/filesys.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/filesys.c b/ext/kenlm/jam-files/engine/filesys.c
deleted file mode 100644
index dadaef8..0000000
--- a/ext/kenlm/jam-files/engine/filesys.c
+++ /dev/null
@@ -1,326 +0,0 @@
-/*
- * Copyright 2001-2004 David Abrahams.
- * Copyright 2005 Rene Rivera.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * filesys.c - OS independant file system manipulation support
- *
- * External routines:
- * file_build1() - construct a path string based on PATHNAME information
- * file_dirscan() - scan a directory for files
- * file_done() - module cleanup called on shutdown
- * file_info() - return cached information about a path
- * file_is_file() - return whether a path identifies an existing file
- * file_query() - get cached information about a path, query the OS if
- * needed
- * file_remove_atexit() - schedule a path to be removed on program exit
- * file_time() - get a file timestamp
- *
- * External routines - utilites for OS specific module implementations:
- * file_query_posix_() - query information about a path using POSIX stat()
- *
- * Internal routines:
- * file_dirscan_impl() - no-profiling worker for file_dirscan()
- */
-
-
-#include "jam.h"
-#include "filesys.h"
-
-#include "lists.h"
-#include "object.h"
-#include "pathsys.h"
-#include "strings.h"
-
-#include <assert.h>
-#include <sys/stat.h>
-
-
-/* Internal OS specific implementation details - have names ending with an
- * underscore and are expected to be implemented in an OS specific fileXXX.c
- * module.
- */
-void file_dirscan_( file_info_t * const dir, scanback func, void * closure );
-int file_collect_dir_content_( file_info_t * const dir );
-void file_query_( file_info_t * const );
-
-static void file_dirscan_impl( OBJECT * dir, scanback func, void * closure );
-static void free_file_info( void * xfile, void * data );
-static void remove_files_atexit( void );
-
-
-static struct hash * filecache_hash;
-
-
-/*
- * file_build1() - construct a path string based on PATHNAME information
- */
-
-void file_build1( PATHNAME * const f, string * file )
-{
- if ( DEBUG_SEARCH )
- {
- printf( "build file: " );
- if ( f->f_root.len )
- printf( "root = '%.*s' ", f->f_root.len, f->f_root.ptr );
- if ( f->f_dir.len )
- printf( "dir = '%.*s' ", f->f_dir.len, f->f_dir.ptr );
- if ( f->f_base.len )
- printf( "base = '%.*s' ", f->f_base.len, f->f_base.ptr );
- printf( "\n" );
- }
-
- /* Start with the grist. If the current grist is not surrounded by <>'s, add
- * them.
- */
- if ( f->f_grist.len )
- {
- if ( f->f_grist.ptr[ 0 ] != '<' )
- string_push_back( file, '<' );
- string_append_range(
- file, f->f_grist.ptr, f->f_grist.ptr + f->f_grist.len );
- if ( file->value[ file->size - 1 ] != '>' )
- string_push_back( file, '>' );
- }
-}
-
-
-/*
- * file_dirscan() - scan a directory for files
- */
-
-void file_dirscan( OBJECT * dir, scanback func, void * closure )
-{
- PROFILE_ENTER( FILE_DIRSCAN );
- file_dirscan_impl( dir, func, closure );
- PROFILE_EXIT( FILE_DIRSCAN );
-}
-
-
-/*
- * file_done() - module cleanup called on shutdown
- */
-
-void file_done()
-{
- remove_files_atexit();
- if ( filecache_hash )
- {
- hashenumerate( filecache_hash, free_file_info, (void *)0 );
- hashdone( filecache_hash );
- }
-}
-
-
-/*
- * file_info() - return cached information about a path
- *
- * Returns a default initialized structure containing only the path's normalized
- * name in case this is the first time this file system entity has been
- * referenced.
- */
-
-file_info_t * file_info( OBJECT * const path, int * found )
-{
- OBJECT * const path_key = path_as_key( path );
- file_info_t * finfo;
-
- if ( !filecache_hash )
- filecache_hash = hashinit( sizeof( file_info_t ), "file_info" );
-
- finfo = (file_info_t *)hash_insert( filecache_hash, path_key, found );
- if ( !*found )
- {
- finfo->name = path_key;
- finfo->files = L0;
- }
- else
- object_free( path_key );
-
- return finfo;
-}
-
-
-/*
- * file_is_file() - return whether a path identifies an existing file
- */
-
-int file_is_file( OBJECT * const path )
-{
- file_info_t const * const ff = file_query( path );
- return ff ? ff->is_file : -1;
-}
-
-
-/*
- * file_time() - get a file timestamp
- */
-
-int file_time( OBJECT * const path, timestamp * const time )
-{
- file_info_t const * const ff = file_query( path );
- if ( !ff ) return -1;
- timestamp_copy( time, &ff->time );
- return 0;
-}
-
-
-/*
- * file_query() - get cached information about a path, query the OS if needed
- *
- * Returns 0 in case querying the OS about the given path fails, e.g. because
- * the path does not reference an existing file system object.
- */
-
-file_info_t * file_query( OBJECT * const path )
-{
- /* FIXME: Add tracking for disappearing files (i.e. those that can not be
- * detected by stat() even though they had been detected successfully
- * before) and see how they should be handled in the rest of Boost Jam code.
- * Possibly allow Jamfiles to specify some files as 'volatile' which would
- * make Boost Jam avoid caching information about those files and instead
- * ask the OS about them every time.
- */
- int found;
- file_info_t * const ff = file_info( path, &found );
- if ( !found )
- {
- file_query_( ff );
- if ( ff->exists )
- {
- /* Set the path's timestamp to 1 in case it is 0 or undetected to avoid
- * confusion with non-existing paths.
- */
- if ( timestamp_empty( &ff->time ) )
- timestamp_init( &ff->time, 1, 0 );
- }
- }
- if ( !ff->exists )
- {
- return 0;
- }
- return ff;
-}
-
-
-/*
- * file_query_posix_() - query information about a path using POSIX stat()
- *
- * Fallback file_query_() implementation for OS specific modules.
- *
- * Note that the Windows POSIX stat() function implementation suffers from
- * several issues:
- * * Does not support file timestamps with resolution finer than 1 second,
- * meaning it can not be used to detect file timestamp changes of less than
- * 1 second. One possible consequence is that some fast-paced touch commands
- * (such as those done by Boost Build's internal testing system if it does
- * not do some extra waiting) will not be detected correctly by the build
- * system.
- * * Returns file modification times automatically adjusted for daylight
- * savings time even though daylight savings time should have nothing to do
- * with internal time representation.
- */
-
-void file_query_posix_( file_info_t * const info )
-{
- struct stat statbuf;
- char const * const pathstr = object_str( info->name );
- char const * const pathspec = *pathstr ? pathstr : ".";
-
- if ( stat( pathspec, &statbuf ) < 0 )
- {
- info->is_file = 0;
- info->is_dir = 0;
- info->exists = 0;
- timestamp_clear( &info->time );
- }
- else
- {
- info->is_file = statbuf.st_mode & S_IFREG ? 1 : 0;
- info->is_dir = statbuf.st_mode & S_IFDIR ? 1 : 0;
- info->exists = 1;
- timestamp_init( &info->time, statbuf.st_mtime, 0 );
- }
-}
-
-
-/*
- * file_remove_atexit() - schedule a path to be removed on program exit
- */
-
-static LIST * files_to_remove = L0;
-
-void file_remove_atexit( OBJECT * const path )
-{
- files_to_remove = list_push_back( files_to_remove, object_copy( path ) );
-}
-
-
-/*
- * file_dirscan_impl() - no-profiling worker for file_dirscan()
- */
-
-static void file_dirscan_impl( OBJECT * dir, scanback func, void * closure )
-{
- file_info_t * const d = file_query( dir );
- if ( !d || !d->is_dir )
- return;
-
- /* Lazy collect the directory content information. */
- if ( list_empty( d->files ) )
- {
- if ( DEBUG_BINDSCAN )
- printf( "scan directory %s\n", object_str( d->name ) );
- if ( file_collect_dir_content_( d ) < 0 )
- return;
- }
-
- /* OS specific part of the file_dirscan operation. */
- file_dirscan_( d, func, closure );
-
- /* Report the collected directory content. */
- {
- LISTITER iter = list_begin( d->files );
- LISTITER const end = list_end( d->files );
- for ( ; iter != end; iter = list_next( iter ) )
- {
- OBJECT * const path = list_item( iter );
- file_info_t const * const ffq = file_query( path );
- /* Using a file name read from a file_info_t structure allows OS
- * specific implementations to store some kind of a normalized file
- * name there. Using such a normalized file name then allows us to
- * correctly recognize different file paths actually identifying the
- * same file. For instance, an implementation may:
- * - convert all file names internally to lower case on a case
- * insensitive file system
- * - convert the NTFS paths to their long path variants as that
- * file system each file system entity may have a long and a
- * short path variant thus allowing for many different path
- * strings identifying the same file.
- */
- (*func)( closure, ffq->name, 1 /* stat()'ed */, &ffq->time );
- }
- }
-}
-
-
-static void free_file_info( void * xfile, void * data )
-{
- file_info_t * const file = (file_info_t *)xfile;
- object_free( file->name );
- list_free( file->files );
-}
-
-
-static void remove_files_atexit( void )
-{
- LISTITER iter = list_begin( files_to_remove );
- LISTITER const end = list_end( files_to_remove );
- for ( ; iter != end; iter = list_next( iter ) )
- remove( object_str( list_item( iter ) ) );
- list_free( files_to_remove );
- files_to_remove = L0;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/filesys.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/filesys.h b/ext/kenlm/jam-files/engine/filesys.h
deleted file mode 100644
index 74fa395..0000000
--- a/ext/kenlm/jam-files/engine/filesys.h
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright 1993-2002 Christopher Seiwald and Perforce Software, Inc.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * filesys.h - OS specific file routines
- */
-
-#ifndef FILESYS_DWA20011025_H
-#define FILESYS_DWA20011025_H
-
-#include "hash.h"
-#include "lists.h"
-#include "object.h"
-#include "pathsys.h"
-#include "timestamp.h"
-
-
-typedef struct file_info_t
-{
- OBJECT * name;
- char is_file;
- char is_dir;
- char exists;
- timestamp time;
- LIST * files;
-} file_info_t;
-
-typedef void (*scanback)( void * closure, OBJECT * path, int found,
- timestamp const * const );
-
-
-void file_archscan( char const * arch, scanback func, void * closure );
-void file_build1( PATHNAME * const f, string * file ) ;
-void file_dirscan( OBJECT * dir, scanback func, void * closure );
-file_info_t * file_info( OBJECT * const path, int * found );
-int file_is_file( OBJECT * const path );
-int file_mkdir( char const * const path );
-file_info_t * file_query( OBJECT * const path );
-void file_remove_atexit( OBJECT * const path );
-void file_supported_fmt_resolution( timestamp * const );
-int file_time( OBJECT * const path, timestamp * const );
-
-/* Internal utility worker functions. */
-void file_query_posix_( file_info_t * const );
-
-void file_done();
-
-#endif
[50/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/.gitignore b/ext/kenlm/.gitignore
deleted file mode 100644
index 6baae3e..0000000
--- a/ext/kenlm/.gitignore
+++ /dev/null
@@ -1,32 +0,0 @@
-bin/
-lm/bin/
-util/bin/
-util/file_piece.cc.gz
-lib/
-dist
-*.swp
-*.o
-query
-build_binary
-kenlm_max_order
-lm/left_test
-lm/model_test
-util/bit_packing_test
-util/file_piece_test
-util/joint_sort_test
-util/probing_hash_table_test
-util/read_compressed_test
-util/sorted_uniform_test
-previous.sh
-jam-files/bjam-1_55
-jam-files/engine/bin.*/
-jam-files/engine/bootstrap/
-doc/
-._*
-include/
-windows/Win32
-windows/x64
-windows/*.user
-windows/*.sdf
-windows/*.opensdf
-windows/*.suo
diff --git a/ext/kenlm/BUILDING b/ext/kenlm/BUILDING
deleted file mode 100644
index e706815..0000000
--- a/ext/kenlm/BUILDING
+++ /dev/null
@@ -1,10 +0,0 @@
-If you have Boost >= 1.36.0 installed:
- ./bjam
- ./bjam install --prefix=/usr #optional
-
-If you only want the query code and do not care about compression (.gz, .bz2, and .xz):
- ./compile_query_only.sh
-
-Windows:
- The windows directory has visual studio files. Note that you need to compile
- the kenlm project before build_binary and ngram_query projects.
diff --git a/ext/kenlm/CMakeLists.txt b/ext/kenlm/CMakeLists.txt
deleted file mode 100644
index 15d9931..0000000
--- a/ext/kenlm/CMakeLists.txt
+++ /dev/null
@@ -1,57 +0,0 @@
-cmake_minimum_required(VERSION 2.8.8)
-#
-# The KenLM cmake files make use of add_library(... OBJECTS ...)
-#
-# This syntax allows grouping of source files when compiling
-# (effectively creating "fake" libraries based on source subdirs).
-#
-# This syntax was only added in cmake version 2.8.8
-#
-# see http://www.cmake.org/Wiki/CMake/Tutorials/Object_Library
-
-
-# This CMake file was created by Lane Schwartz <do...@gmail.com>
-
-
-# Define a single cmake project
-project(kenlm)
-
-# Compile all executables into bin/
-set(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/bin)
-
-# Compile all libraries into lib/
-set(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/lib)
-
-# Tell cmake that we want unit tests to be compiled
-include(CTest)
-enable_testing()
-
-# Add our CMake helper functions
-include(cmake/KenLMFunctions.cmake)
-
-# And our helper modules
-list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/modules)
-
-# We need boost
-find_package(Boost 1.36.0 REQUIRED COMPONENTS
- program_options
- system
- thread
- unit_test_framework
-)
-
-
-
-
-# Define where include files live
-include_directories(
- ${PROJECT_SOURCE_DIR}
- ${Boost_INCLUDE_DIRS}
-)
-
-option(BUILD_INTERPOLATE "Build language model interpolation code" OFF)
-
-# Process subdirectories
-add_subdirectory(util)
-add_subdirectory(lm)
-
diff --git a/ext/kenlm/COPYING b/ext/kenlm/COPYING
deleted file mode 100644
index 4362b49..0000000
--- a/ext/kenlm/COPYING
+++ /dev/null
@@ -1,502 +0,0 @@
- GNU LESSER GENERAL PUBLIC LICENSE
- Version 2.1, February 1999
-
- Copyright (C) 1991, 1999 Free Software Foundation, Inc.
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-[This is the first released version of the Lesser GPL. It also counts
- as the successor of the GNU Library Public License, version 2, hence
- the version number 2.1.]
-
- Preamble
-
- The licenses for most software are designed to take away your
-freedom to share and change it. By contrast, the GNU General Public
-Licenses are intended to guarantee your freedom to share and change
-free software--to make sure the software is free for all its users.
-
- This license, the Lesser General Public License, applies to some
-specially designated software packages--typically libraries--of the
-Free Software Foundation and other authors who decide to use it. You
-can use it too, but we suggest you first think carefully about whether
-this license or the ordinary General Public License is the better
-strategy to use in any particular case, based on the explanations below.
-
- When we speak of free software, we are referring to freedom of use,
-not price. Our General Public Licenses are designed to make sure that
-you have the freedom to distribute copies of free software (and charge
-for this service if you wish); that you receive source code or can get
-it if you want it; that you can change the software and use pieces of
-it in new free programs; and that you are informed that you can do
-these things.
-
- To protect your rights, we need to make restrictions that forbid
-distributors to deny you these rights or to ask you to surrender these
-rights. These restrictions translate to certain responsibilities for
-you if you distribute copies of the library or if you modify it.
-
- For example, if you distribute copies of the library, whether gratis
-or for a fee, you must give the recipients all the rights that we gave
-you. You must make sure that they, too, receive or can get the source
-code. If you link other code with the library, you must provide
-complete object files to the recipients, so that they can relink them
-with the library after making changes to the library and recompiling
-it. And you must show them these terms so they know their rights.
-
- We protect your rights with a two-step method: (1) we copyright the
-library, and (2) we offer you this license, which gives you legal
-permission to copy, distribute and/or modify the library.
-
- To protect each distributor, we want to make it very clear that
-there is no warranty for the free library. Also, if the library is
-modified by someone else and passed on, the recipients should know
-that what they have is not the original version, so that the original
-author's reputation will not be affected by problems that might be
-introduced by others.
-
- Finally, software patents pose a constant threat to the existence of
-any free program. We wish to make sure that a company cannot
-effectively restrict the users of a free program by obtaining a
-restrictive license from a patent holder. Therefore, we insist that
-any patent license obtained for a version of the library must be
-consistent with the full freedom of use specified in this license.
-
- Most GNU software, including some libraries, is covered by the
-ordinary GNU General Public License. This license, the GNU Lesser
-General Public License, applies to certain designated libraries, and
-is quite different from the ordinary General Public License. We use
-this license for certain libraries in order to permit linking those
-libraries into non-free programs.
-
- When a program is linked with a library, whether statically or using
-a shared library, the combination of the two is legally speaking a
-combined work, a derivative of the original library. The ordinary
-General Public License therefore permits such linking only if the
-entire combination fits its criteria of freedom. The Lesser General
-Public License permits more lax criteria for linking other code with
-the library.
-
- We call this license the "Lesser" General Public License because it
-does Less to protect the user's freedom than the ordinary General
-Public License. It also provides other free software developers Less
-of an advantage over competing non-free programs. These disadvantages
-are the reason we use the ordinary General Public License for many
-libraries. However, the Lesser license provides advantages in certain
-special circumstances.
-
- For example, on rare occasions, there may be a special need to
-encourage the widest possible use of a certain library, so that it becomes
-a de-facto standard. To achieve this, non-free programs must be
-allowed to use the library. A more frequent case is that a free
-library does the same job as widely used non-free libraries. In this
-case, there is little to gain by limiting the free library to free
-software only, so we use the Lesser General Public License.
-
- In other cases, permission to use a particular library in non-free
-programs enables a greater number of people to use a large body of
-free software. For example, permission to use the GNU C Library in
-non-free programs enables many more people to use the whole GNU
-operating system, as well as its variant, the GNU/Linux operating
-system.
-
- Although the Lesser General Public License is Less protective of the
-users' freedom, it does ensure that the user of a program that is
-linked with the Library has the freedom and the wherewithal to run
-that program using a modified version of the Library.
-
- The precise terms and conditions for copying, distribution and
-modification follow. Pay close attention to the difference between a
-"work based on the library" and a "work that uses the library". The
-former contains code derived from the library, whereas the latter must
-be combined with the library in order to run.
-
- GNU LESSER GENERAL PUBLIC LICENSE
- TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
-
- 0. This License Agreement applies to any software library or other
-program which contains a notice placed by the copyright holder or
-other authorized party saying it may be distributed under the terms of
-this Lesser General Public License (also called "this License").
-Each licensee is addressed as "you".
-
- A "library" means a collection of software functions and/or data
-prepared so as to be conveniently linked with application programs
-(which use some of those functions and data) to form executables.
-
- The "Library", below, refers to any such software library or work
-which has been distributed under these terms. A "work based on the
-Library" means either the Library or any derivative work under
-copyright law: that is to say, a work containing the Library or a
-portion of it, either verbatim or with modifications and/or translated
-straightforwardly into another language. (Hereinafter, translation is
-included without limitation in the term "modification".)
-
- "Source code" for a work means the preferred form of the work for
-making modifications to it. For a library, complete source code means
-all the source code for all modules it contains, plus any associated
-interface definition files, plus the scripts used to control compilation
-and installation of the library.
-
- Activities other than copying, distribution and modification are not
-covered by this License; they are outside its scope. The act of
-running a program using the Library is not restricted, and output from
-such a program is covered only if its contents constitute a work based
-on the Library (independent of the use of the Library in a tool for
-writing it). Whether that is true depends on what the Library does
-and what the program that uses the Library does.
-
- 1. You may copy and distribute verbatim copies of the Library's
-complete source code as you receive it, in any medium, provided that
-you conspicuously and appropriately publish on each copy an
-appropriate copyright notice and disclaimer of warranty; keep intact
-all the notices that refer to this License and to the absence of any
-warranty; and distribute a copy of this License along with the
-Library.
-
- You may charge a fee for the physical act of transferring a copy,
-and you may at your option offer warranty protection in exchange for a
-fee.
-
- 2. You may modify your copy or copies of the Library or any portion
-of it, thus forming a work based on the Library, and copy and
-distribute such modifications or work under the terms of Section 1
-above, provided that you also meet all of these conditions:
-
- a) The modified work must itself be a software library.
-
- b) You must cause the files modified to carry prominent notices
- stating that you changed the files and the date of any change.
-
- c) You must cause the whole of the work to be licensed at no
- charge to all third parties under the terms of this License.
-
- d) If a facility in the modified Library refers to a function or a
- table of data to be supplied by an application program that uses
- the facility, other than as an argument passed when the facility
- is invoked, then you must make a good faith effort to ensure that,
- in the event an application does not supply such function or
- table, the facility still operates, and performs whatever part of
- its purpose remains meaningful.
-
- (For example, a function in a library to compute square roots has
- a purpose that is entirely well-defined independent of the
- application. Therefore, Subsection 2d requires that any
- application-supplied function or table used by this function must
- be optional: if the application does not supply it, the square
- root function must still compute square roots.)
-
-These requirements apply to the modified work as a whole. If
-identifiable sections of that work are not derived from the Library,
-and can be reasonably considered independent and separate works in
-themselves, then this License, and its terms, do not apply to those
-sections when you distribute them as separate works. But when you
-distribute the same sections as part of a whole which is a work based
-on the Library, the distribution of the whole must be on the terms of
-this License, whose permissions for other licensees extend to the
-entire whole, and thus to each and every part regardless of who wrote
-it.
-
-Thus, it is not the intent of this section to claim rights or contest
-your rights to work written entirely by you; rather, the intent is to
-exercise the right to control the distribution of derivative or
-collective works based on the Library.
-
-In addition, mere aggregation of another work not based on the Library
-with the Library (or with a work based on the Library) on a volume of
-a storage or distribution medium does not bring the other work under
-the scope of this License.
-
- 3. You may opt to apply the terms of the ordinary GNU General Public
-License instead of this License to a given copy of the Library. To do
-this, you must alter all the notices that refer to this License, so
-that they refer to the ordinary GNU General Public License, version 2,
-instead of to this License. (If a newer version than version 2 of the
-ordinary GNU General Public License has appeared, then you can specify
-that version instead if you wish.) Do not make any other change in
-these notices.
-
- Once this change is made in a given copy, it is irreversible for
-that copy, so the ordinary GNU General Public License applies to all
-subsequent copies and derivative works made from that copy.
-
- This option is useful when you wish to copy part of the code of
-the Library into a program that is not a library.
-
- 4. You may copy and distribute the Library (or a portion or
-derivative of it, under Section 2) in object code or executable form
-under the terms of Sections 1 and 2 above provided that you accompany
-it with the complete corresponding machine-readable source code, which
-must be distributed under the terms of Sections 1 and 2 above on a
-medium customarily used for software interchange.
-
- If distribution of object code is made by offering access to copy
-from a designated place, then offering equivalent access to copy the
-source code from the same place satisfies the requirement to
-distribute the source code, even though third parties are not
-compelled to copy the source along with the object code.
-
- 5. A program that contains no derivative of any portion of the
-Library, but is designed to work with the Library by being compiled or
-linked with it, is called a "work that uses the Library". Such a
-work, in isolation, is not a derivative work of the Library, and
-therefore falls outside the scope of this License.
-
- However, linking a "work that uses the Library" with the Library
-creates an executable that is a derivative of the Library (because it
-contains portions of the Library), rather than a "work that uses the
-library". The executable is therefore covered by this License.
-Section 6 states terms for distribution of such executables.
-
- When a "work that uses the Library" uses material from a header file
-that is part of the Library, the object code for the work may be a
-derivative work of the Library even though the source code is not.
-Whether this is true is especially significant if the work can be
-linked without the Library, or if the work is itself a library. The
-threshold for this to be true is not precisely defined by law.
-
- If such an object file uses only numerical parameters, data
-structure layouts and accessors, and small macros and small inline
-functions (ten lines or less in length), then the use of the object
-file is unrestricted, regardless of whether it is legally a derivative
-work. (Executables containing this object code plus portions of the
-Library will still fall under Section 6.)
-
- Otherwise, if the work is a derivative of the Library, you may
-distribute the object code for the work under the terms of Section 6.
-Any executables containing that work also fall under Section 6,
-whether or not they are linked directly with the Library itself.
-
- 6. As an exception to the Sections above, you may also combine or
-link a "work that uses the Library" with the Library to produce a
-work containing portions of the Library, and distribute that work
-under terms of your choice, provided that the terms permit
-modification of the work for the customer's own use and reverse
-engineering for debugging such modifications.
-
- You must give prominent notice with each copy of the work that the
-Library is used in it and that the Library and its use are covered by
-this License. You must supply a copy of this License. If the work
-during execution displays copyright notices, you must include the
-copyright notice for the Library among them, as well as a reference
-directing the user to the copy of this License. Also, you must do one
-of these things:
-
- a) Accompany the work with the complete corresponding
- machine-readable source code for the Library including whatever
- changes were used in the work (which must be distributed under
- Sections 1 and 2 above); and, if the work is an executable linked
- with the Library, with the complete machine-readable "work that
- uses the Library", as object code and/or source code, so that the
- user can modify the Library and then relink to produce a modified
- executable containing the modified Library. (It is understood
- that the user who changes the contents of definitions files in the
- Library will not necessarily be able to recompile the application
- to use the modified definitions.)
-
- b) Use a suitable shared library mechanism for linking with the
- Library. A suitable mechanism is one that (1) uses at run time a
- copy of the library already present on the user's computer system,
- rather than copying library functions into the executable, and (2)
- will operate properly with a modified version of the library, if
- the user installs one, as long as the modified version is
- interface-compatible with the version that the work was made with.
-
- c) Accompany the work with a written offer, valid for at
- least three years, to give the same user the materials
- specified in Subsection 6a, above, for a charge no more
- than the cost of performing this distribution.
-
- d) If distribution of the work is made by offering access to copy
- from a designated place, offer equivalent access to copy the above
- specified materials from the same place.
-
- e) Verify that the user has already received a copy of these
- materials or that you have already sent this user a copy.
-
- For an executable, the required form of the "work that uses the
-Library" must include any data and utility programs needed for
-reproducing the executable from it. However, as a special exception,
-the materials to be distributed need not include anything that is
-normally distributed (in either source or binary form) with the major
-components (compiler, kernel, and so on) of the operating system on
-which the executable runs, unless that component itself accompanies
-the executable.
-
- It may happen that this requirement contradicts the license
-restrictions of other proprietary libraries that do not normally
-accompany the operating system. Such a contradiction means you cannot
-use both them and the Library together in an executable that you
-distribute.
-
- 7. You may place library facilities that are a work based on the
-Library side-by-side in a single library together with other library
-facilities not covered by this License, and distribute such a combined
-library, provided that the separate distribution of the work based on
-the Library and of the other library facilities is otherwise
-permitted, and provided that you do these two things:
-
- a) Accompany the combined library with a copy of the same work
- based on the Library, uncombined with any other library
- facilities. This must be distributed under the terms of the
- Sections above.
-
- b) Give prominent notice with the combined library of the fact
- that part of it is a work based on the Library, and explaining
- where to find the accompanying uncombined form of the same work.
-
- 8. You may not copy, modify, sublicense, link with, or distribute
-the Library except as expressly provided under this License. Any
-attempt otherwise to copy, modify, sublicense, link with, or
-distribute the Library is void, and will automatically terminate your
-rights under this License. However, parties who have received copies,
-or rights, from you under this License will not have their licenses
-terminated so long as such parties remain in full compliance.
-
- 9. You are not required to accept this License, since you have not
-signed it. However, nothing else grants you permission to modify or
-distribute the Library or its derivative works. These actions are
-prohibited by law if you do not accept this License. Therefore, by
-modifying or distributing the Library (or any work based on the
-Library), you indicate your acceptance of this License to do so, and
-all its terms and conditions for copying, distributing or modifying
-the Library or works based on it.
-
- 10. Each time you redistribute the Library (or any work based on the
-Library), the recipient automatically receives a license from the
-original licensor to copy, distribute, link with or modify the Library
-subject to these terms and conditions. You may not impose any further
-restrictions on the recipients' exercise of the rights granted herein.
-You are not responsible for enforcing compliance by third parties with
-this License.
-
- 11. If, as a consequence of a court judgment or allegation of patent
-infringement or for any other reason (not limited to patent issues),
-conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License. If you cannot
-distribute so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you
-may not distribute the Library at all. For example, if a patent
-license would not permit royalty-free redistribution of the Library by
-all those who receive copies directly or indirectly through you, then
-the only way you could satisfy both it and this License would be to
-refrain entirely from distribution of the Library.
-
-If any portion of this section is held invalid or unenforceable under any
-particular circumstance, the balance of the section is intended to apply,
-and the section as a whole is intended to apply in other circumstances.
-
-It is not the purpose of this section to induce you to infringe any
-patents or other property right claims or to contest validity of any
-such claims; this section has the sole purpose of protecting the
-integrity of the free software distribution system which is
-implemented by public license practices. Many people have made
-generous contributions to the wide range of software distributed
-through that system in reliance on consistent application of that
-system; it is up to the author/donor to decide if he or she is willing
-to distribute software through any other system and a licensee cannot
-impose that choice.
-
-This section is intended to make thoroughly clear what is believed to
-be a consequence of the rest of this License.
-
- 12. If the distribution and/or use of the Library is restricted in
-certain countries either by patents or by copyrighted interfaces, the
-original copyright holder who places the Library under this License may add
-an explicit geographical distribution limitation excluding those countries,
-so that distribution is permitted only in or among countries not thus
-excluded. In such case, this License incorporates the limitation as if
-written in the body of this License.
-
- 13. The Free Software Foundation may publish revised and/or new
-versions of the Lesser General Public License from time to time.
-Such new versions will be similar in spirit to the present version,
-but may differ in detail to address new problems or concerns.
-
-Each version is given a distinguishing version number. If the Library
-specifies a version number of this License which applies to it and
-"any later version", you have the option of following the terms and
-conditions either of that version or of any later version published by
-the Free Software Foundation. If the Library does not specify a
-license version number, you may choose any version ever published by
-the Free Software Foundation.
-
- 14. If you wish to incorporate parts of the Library into other free
-programs whose distribution conditions are incompatible with these,
-write to the author to ask for permission. For software which is
-copyrighted by the Free Software Foundation, write to the Free
-Software Foundation; we sometimes make exceptions for this. Our
-decision will be guided by the two goals of preserving the free status
-of all derivatives of our free software and of promoting the sharing
-and reuse of software generally.
-
- NO WARRANTY
-
- 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
-WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
-EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
-OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
-KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
-LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
-THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
- 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
-WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
-AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
-FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
-CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
-LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
-RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
-FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
-SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
-DAMAGES.
-
- END OF TERMS AND CONDITIONS
-
- How to Apply These Terms to Your New Libraries
-
- If you develop a new library, and you want it to be of the greatest
-possible use to the public, we recommend making it free software that
-everyone can redistribute and change. You can do so by permitting
-redistribution under these terms (or, alternatively, under the terms of the
-ordinary General Public License).
-
- To apply these terms, attach the following notices to the library. It is
-safest to attach them to the start of each source file to most effectively
-convey the exclusion of warranty; and each file should have at least the
-"copyright" line and a pointer to where the full notice is found.
-
- <one line to give the library's name and a brief idea of what it does.>
- Copyright (C) <year> <name of author>
-
- This library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- This library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with this library; if not, write to the Free Software
- Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-Also add information on how to contact you by electronic and paper mail.
-
-You should also get your employer (if you work as a programmer) or your
-school, if any, to sign a "copyright disclaimer" for the library, if
-necessary. Here is a sample; alter the names:
-
- Yoyodyne, Inc., hereby disclaims all copyright interest in the
- library `Frob' (a library for tweaking knobs) written by James Random Hacker.
-
- <signature of Ty Coon>, 1 April 1990
- Ty Coon, President of Vice
-
-That's all there is to it!
diff --git a/ext/kenlm/COPYING.3 b/ext/kenlm/COPYING.3
deleted file mode 100644
index 94a9ed0..0000000
--- a/ext/kenlm/COPYING.3
+++ /dev/null
@@ -1,674 +0,0 @@
- GNU GENERAL PUBLIC LICENSE
- Version 3, 29 June 2007
-
- Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
- Preamble
-
- The GNU General Public License is a free, copyleft license for
-software and other kinds of works.
-
- The licenses for most software and other practical works are designed
-to take away your freedom to share and change the works. By contrast,
-the GNU General Public License is intended to guarantee your freedom to
-share and change all versions of a program--to make sure it remains free
-software for all its users. We, the Free Software Foundation, use the
-GNU General Public License for most of our software; it applies also to
-any other work released this way by its authors. You can apply it to
-your programs, too.
-
- When we speak of free software, we are referring to freedom, not
-price. Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-them if you wish), that you receive source code or can get it if you
-want it, that you can change the software or use pieces of it in new
-free programs, and that you know you can do these things.
-
- To protect your rights, we need to prevent others from denying you
-these rights or asking you to surrender the rights. Therefore, you have
-certain responsibilities if you distribute copies of the software, or if
-you modify it: responsibilities to respect the freedom of others.
-
- For example, if you distribute copies of such a program, whether
-gratis or for a fee, you must pass on to the recipients the same
-freedoms that you received. You must make sure that they, too, receive
-or can get the source code. And you must show them these terms so they
-know their rights.
-
- Developers that use the GNU GPL protect your rights with two steps:
-(1) assert copyright on the software, and (2) offer you this License
-giving you legal permission to copy, distribute and/or modify it.
-
- For the developers' and authors' protection, the GPL clearly explains
-that there is no warranty for this free software. For both users' and
-authors' sake, the GPL requires that modified versions be marked as
-changed, so that their problems will not be attributed erroneously to
-authors of previous versions.
-
- Some devices are designed to deny users access to install or run
-modified versions of the software inside them, although the manufacturer
-can do so. This is fundamentally incompatible with the aim of
-protecting users' freedom to change the software. The systematic
-pattern of such abuse occurs in the area of products for individuals to
-use, which is precisely where it is most unacceptable. Therefore, we
-have designed this version of the GPL to prohibit the practice for those
-products. If such problems arise substantially in other domains, we
-stand ready to extend this provision to those domains in future versions
-of the GPL, as needed to protect the freedom of users.
-
- Finally, every program is threatened constantly by software patents.
-States should not allow patents to restrict development and use of
-software on general-purpose computers, but in those that do, we wish to
-avoid the special danger that patents applied to a free program could
-make it effectively proprietary. To prevent this, the GPL assures that
-patents cannot be used to render the program non-free.
-
- The precise terms and conditions for copying, distribution and
-modification follow.
-
- TERMS AND CONDITIONS
-
- 0. Definitions.
-
- "This License" refers to version 3 of the GNU General Public License.
-
- "Copyright" also means copyright-like laws that apply to other kinds of
-works, such as semiconductor masks.
-
- "The Program" refers to any copyrightable work licensed under this
-License. Each licensee is addressed as "you". "Licensees" and
-"recipients" may be individuals or organizations.
-
- To "modify" a work means to copy from or adapt all or part of the work
-in a fashion requiring copyright permission, other than the making of an
-exact copy. The resulting work is called a "modified version" of the
-earlier work or a work "based on" the earlier work.
-
- A "covered work" means either the unmodified Program or a work based
-on the Program.
-
- To "propagate" a work means to do anything with it that, without
-permission, would make you directly or secondarily liable for
-infringement under applicable copyright law, except executing it on a
-computer or modifying a private copy. Propagation includes copying,
-distribution (with or without modification), making available to the
-public, and in some countries other activities as well.
-
- To "convey" a work means any kind of propagation that enables other
-parties to make or receive copies. Mere interaction with a user through
-a computer network, with no transfer of a copy, is not conveying.
-
- An interactive user interface displays "Appropriate Legal Notices"
-to the extent that it includes a convenient and prominently visible
-feature that (1) displays an appropriate copyright notice, and (2)
-tells the user that there is no warranty for the work (except to the
-extent that warranties are provided), that licensees may convey the
-work under this License, and how to view a copy of this License. If
-the interface presents a list of user commands or options, such as a
-menu, a prominent item in the list meets this criterion.
-
- 1. Source Code.
-
- The "source code" for a work means the preferred form of the work
-for making modifications to it. "Object code" means any non-source
-form of a work.
-
- A "Standard Interface" means an interface that either is an official
-standard defined by a recognized standards body, or, in the case of
-interfaces specified for a particular programming language, one that
-is widely used among developers working in that language.
-
- The "System Libraries" of an executable work include anything, other
-than the work as a whole, that (a) is included in the normal form of
-packaging a Major Component, but which is not part of that Major
-Component, and (b) serves only to enable use of the work with that
-Major Component, or to implement a Standard Interface for which an
-implementation is available to the public in source code form. A
-"Major Component", in this context, means a major essential component
-(kernel, window system, and so on) of the specific operating system
-(if any) on which the executable work runs, or a compiler used to
-produce the work, or an object code interpreter used to run it.
-
- The "Corresponding Source" for a work in object code form means all
-the source code needed to generate, install, and (for an executable
-work) run the object code and to modify the work, including scripts to
-control those activities. However, it does not include the work's
-System Libraries, or general-purpose tools or generally available free
-programs which are used unmodified in performing those activities but
-which are not part of the work. For example, Corresponding Source
-includes interface definition files associated with source files for
-the work, and the source code for shared libraries and dynamically
-linked subprograms that the work is specifically designed to require,
-such as by intimate data communication or control flow between those
-subprograms and other parts of the work.
-
- The Corresponding Source need not include anything that users
-can regenerate automatically from other parts of the Corresponding
-Source.
-
- The Corresponding Source for a work in source code form is that
-same work.
-
- 2. Basic Permissions.
-
- All rights granted under this License are granted for the term of
-copyright on the Program, and are irrevocable provided the stated
-conditions are met. This License explicitly affirms your unlimited
-permission to run the unmodified Program. The output from running a
-covered work is covered by this License only if the output, given its
-content, constitutes a covered work. This License acknowledges your
-rights of fair use or other equivalent, as provided by copyright law.
-
- You may make, run and propagate covered works that you do not
-convey, without conditions so long as your license otherwise remains
-in force. You may convey covered works to others for the sole purpose
-of having them make modifications exclusively for you, or provide you
-with facilities for running those works, provided that you comply with
-the terms of this License in conveying all material for which you do
-not control copyright. Those thus making or running the covered works
-for you must do so exclusively on your behalf, under your direction
-and control, on terms that prohibit them from making any copies of
-your copyrighted material outside their relationship with you.
-
- Conveying under any other circumstances is permitted solely under
-the conditions stated below. Sublicensing is not allowed; section 10
-makes it unnecessary.
-
- 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
-
- No covered work shall be deemed part of an effective technological
-measure under any applicable law fulfilling obligations under article
-11 of the WIPO copyright treaty adopted on 20 December 1996, or
-similar laws prohibiting or restricting circumvention of such
-measures.
-
- When you convey a covered work, you waive any legal power to forbid
-circumvention of technological measures to the extent such circumvention
-is effected by exercising rights under this License with respect to
-the covered work, and you disclaim any intention to limit operation or
-modification of the work as a means of enforcing, against the work's
-users, your or third parties' legal rights to forbid circumvention of
-technological measures.
-
- 4. Conveying Verbatim Copies.
-
- You may convey verbatim copies of the Program's source code as you
-receive it, in any medium, provided that you conspicuously and
-appropriately publish on each copy an appropriate copyright notice;
-keep intact all notices stating that this License and any
-non-permissive terms added in accord with section 7 apply to the code;
-keep intact all notices of the absence of any warranty; and give all
-recipients a copy of this License along with the Program.
-
- You may charge any price or no price for each copy that you convey,
-and you may offer support or warranty protection for a fee.
-
- 5. Conveying Modified Source Versions.
-
- You may convey a work based on the Program, or the modifications to
-produce it from the Program, in the form of source code under the
-terms of section 4, provided that you also meet all of these conditions:
-
- a) The work must carry prominent notices stating that you modified
- it, and giving a relevant date.
-
- b) The work must carry prominent notices stating that it is
- released under this License and any conditions added under section
- 7. This requirement modifies the requirement in section 4 to
- "keep intact all notices".
-
- c) You must license the entire work, as a whole, under this
- License to anyone who comes into possession of a copy. This
- License will therefore apply, along with any applicable section 7
- additional terms, to the whole of the work, and all its parts,
- regardless of how they are packaged. This License gives no
- permission to license the work in any other way, but it does not
- invalidate such permission if you have separately received it.
-
- d) If the work has interactive user interfaces, each must display
- Appropriate Legal Notices; however, if the Program has interactive
- interfaces that do not display Appropriate Legal Notices, your
- work need not make them do so.
-
- A compilation of a covered work with other separate and independent
-works, which are not by their nature extensions of the covered work,
-and which are not combined with it such as to form a larger program,
-in or on a volume of a storage or distribution medium, is called an
-"aggregate" if the compilation and its resulting copyright are not
-used to limit the access or legal rights of the compilation's users
-beyond what the individual works permit. Inclusion of a covered work
-in an aggregate does not cause this License to apply to the other
-parts of the aggregate.
-
- 6. Conveying Non-Source Forms.
-
- You may convey a covered work in object code form under the terms
-of sections 4 and 5, provided that you also convey the
-machine-readable Corresponding Source under the terms of this License,
-in one of these ways:
-
- a) Convey the object code in, or embodied in, a physical product
- (including a physical distribution medium), accompanied by the
- Corresponding Source fixed on a durable physical medium
- customarily used for software interchange.
-
- b) Convey the object code in, or embodied in, a physical product
- (including a physical distribution medium), accompanied by a
- written offer, valid for at least three years and valid for as
- long as you offer spare parts or customer support for that product
- model, to give anyone who possesses the object code either (1) a
- copy of the Corresponding Source for all the software in the
- product that is covered by this License, on a durable physical
- medium customarily used for software interchange, for a price no
- more than your reasonable cost of physically performing this
- conveying of source, or (2) access to copy the
- Corresponding Source from a network server at no charge.
-
- c) Convey individual copies of the object code with a copy of the
- written offer to provide the Corresponding Source. This
- alternative is allowed only occasionally and noncommercially, and
- only if you received the object code with such an offer, in accord
- with subsection 6b.
-
- d) Convey the object code by offering access from a designated
- place (gratis or for a charge), and offer equivalent access to the
- Corresponding Source in the same way through the same place at no
- further charge. You need not require recipients to copy the
- Corresponding Source along with the object code. If the place to
- copy the object code is a network server, the Corresponding Source
- may be on a different server (operated by you or a third party)
- that supports equivalent copying facilities, provided you maintain
- clear directions next to the object code saying where to find the
- Corresponding Source. Regardless of what server hosts the
- Corresponding Source, you remain obligated to ensure that it is
- available for as long as needed to satisfy these requirements.
-
- e) Convey the object code using peer-to-peer transmission, provided
- you inform other peers where the object code and Corresponding
- Source of the work are being offered to the general public at no
- charge under subsection 6d.
-
- A separable portion of the object code, whose source code is excluded
-from the Corresponding Source as a System Library, need not be
-included in conveying the object code work.
-
- A "User Product" is either (1) a "consumer product", which means any
-tangible personal property which is normally used for personal, family,
-or household purposes, or (2) anything designed or sold for incorporation
-into a dwelling. In determining whether a product is a consumer product,
-doubtful cases shall be resolved in favor of coverage. For a particular
-product received by a particular user, "normally used" refers to a
-typical or common use of that class of product, regardless of the status
-of the particular user or of the way in which the particular user
-actually uses, or expects or is expected to use, the product. A product
-is a consumer product regardless of whether the product has substantial
-commercial, industrial or non-consumer uses, unless such uses represent
-the only significant mode of use of the product.
-
- "Installation Information" for a User Product means any methods,
-procedures, authorization keys, or other information required to install
-and execute modified versions of a covered work in that User Product from
-a modified version of its Corresponding Source. The information must
-suffice to ensure that the continued functioning of the modified object
-code is in no case prevented or interfered with solely because
-modification has been made.
-
- If you convey an object code work under this section in, or with, or
-specifically for use in, a User Product, and the conveying occurs as
-part of a transaction in which the right of possession and use of the
-User Product is transferred to the recipient in perpetuity or for a
-fixed term (regardless of how the transaction is characterized), the
-Corresponding Source conveyed under this section must be accompanied
-by the Installation Information. But this requirement does not apply
-if neither you nor any third party retains the ability to install
-modified object code on the User Product (for example, the work has
-been installed in ROM).
-
- The requirement to provide Installation Information does not include a
-requirement to continue to provide support service, warranty, or updates
-for a work that has been modified or installed by the recipient, or for
-the User Product in which it has been modified or installed. Access to a
-network may be denied when the modification itself materially and
-adversely affects the operation of the network or violates the rules and
-protocols for communication across the network.
-
- Corresponding Source conveyed, and Installation Information provided,
-in accord with this section must be in a format that is publicly
-documented (and with an implementation available to the public in
-source code form), and must require no special password or key for
-unpacking, reading or copying.
-
- 7. Additional Terms.
-
- "Additional permissions" are terms that supplement the terms of this
-License by making exceptions from one or more of its conditions.
-Additional permissions that are applicable to the entire Program shall
-be treated as though they were included in this License, to the extent
-that they are valid under applicable law. If additional permissions
-apply only to part of the Program, that part may be used separately
-under those permissions, but the entire Program remains governed by
-this License without regard to the additional permissions.
-
- When you convey a copy of a covered work, you may at your option
-remove any additional permissions from that copy, or from any part of
-it. (Additional permissions may be written to require their own
-removal in certain cases when you modify the work.) You may place
-additional permissions on material, added by you to a covered work,
-for which you have or can give appropriate copyright permission.
-
- Notwithstanding any other provision of this License, for material you
-add to a covered work, you may (if authorized by the copyright holders of
-that material) supplement the terms of this License with terms:
-
- a) Disclaiming warranty or limiting liability differently from the
- terms of sections 15 and 16 of this License; or
-
- b) Requiring preservation of specified reasonable legal notices or
- author attributions in that material or in the Appropriate Legal
- Notices displayed by works containing it; or
-
- c) Prohibiting misrepresentation of the origin of that material, or
- requiring that modified versions of such material be marked in
- reasonable ways as different from the original version; or
-
- d) Limiting the use for publicity purposes of names of licensors or
- authors of the material; or
-
- e) Declining to grant rights under trademark law for use of some
- trade names, trademarks, or service marks; or
-
- f) Requiring indemnification of licensors and authors of that
- material by anyone who conveys the material (or modified versions of
- it) with contractual assumptions of liability to the recipient, for
- any liability that these contractual assumptions directly impose on
- those licensors and authors.
-
- All other non-permissive additional terms are considered "further
-restrictions" within the meaning of section 10. If the Program as you
-received it, or any part of it, contains a notice stating that it is
-governed by this License along with a term that is a further
-restriction, you may remove that term. If a license document contains
-a further restriction but permits relicensing or conveying under this
-License, you may add to a covered work material governed by the terms
-of that license document, provided that the further restriction does
-not survive such relicensing or conveying.
-
- If you add terms to a covered work in accord with this section, you
-must place, in the relevant source files, a statement of the
-additional terms that apply to those files, or a notice indicating
-where to find the applicable terms.
-
- Additional terms, permissive or non-permissive, may be stated in the
-form of a separately written license, or stated as exceptions;
-the above requirements apply either way.
-
- 8. Termination.
-
- You may not propagate or modify a covered work except as expressly
-provided under this License. Any attempt otherwise to propagate or
-modify it is void, and will automatically terminate your rights under
-this License (including any patent licenses granted under the third
-paragraph of section 11).
-
- However, if you cease all violation of this License, then your
-license from a particular copyright holder is reinstated (a)
-provisionally, unless and until the copyright holder explicitly and
-finally terminates your license, and (b) permanently, if the copyright
-holder fails to notify you of the violation by some reasonable means
-prior to 60 days after the cessation.
-
- Moreover, your license from a particular copyright holder is
-reinstated permanently if the copyright holder notifies you of the
-violation by some reasonable means, this is the first time you have
-received notice of violation of this License (for any work) from that
-copyright holder, and you cure the violation prior to 30 days after
-your receipt of the notice.
-
- Termination of your rights under this section does not terminate the
-licenses of parties who have received copies or rights from you under
-this License. If your rights have been terminated and not permanently
-reinstated, you do not qualify to receive new licenses for the same
-material under section 10.
-
- 9. Acceptance Not Required for Having Copies.
-
- You are not required to accept this License in order to receive or
-run a copy of the Program. Ancillary propagation of a covered work
-occurring solely as a consequence of using peer-to-peer transmission
-to receive a copy likewise does not require acceptance. However,
-nothing other than this License grants you permission to propagate or
-modify any covered work. These actions infringe copyright if you do
-not accept this License. Therefore, by modifying or propagating a
-covered work, you indicate your acceptance of this License to do so.
-
- 10. Automatic Licensing of Downstream Recipients.
-
- Each time you convey a covered work, the recipient automatically
-receives a license from the original licensors, to run, modify and
-propagate that work, subject to this License. You are not responsible
-for enforcing compliance by third parties with this License.
-
- An "entity transaction" is a transaction transferring control of an
-organization, or substantially all assets of one, or subdividing an
-organization, or merging organizations. If propagation of a covered
-work results from an entity transaction, each party to that
-transaction who receives a copy of the work also receives whatever
-licenses to the work the party's predecessor in interest had or could
-give under the previous paragraph, plus a right to possession of the
-Corresponding Source of the work from the predecessor in interest, if
-the predecessor has it or can get it with reasonable efforts.
-
- You may not impose any further restrictions on the exercise of the
-rights granted or affirmed under this License. For example, you may
-not impose a license fee, royalty, or other charge for exercise of
-rights granted under this License, and you may not initiate litigation
-(including a cross-claim or counterclaim in a lawsuit) alleging that
-any patent claim is infringed by making, using, selling, offering for
-sale, or importing the Program or any portion of it.
-
- 11. Patents.
-
- A "contributor" is a copyright holder who authorizes use under this
-License of the Program or a work on which the Program is based. The
-work thus licensed is called the contributor's "contributor version".
-
- A contributor's "essential patent claims" are all patent claims
-owned or controlled by the contributor, whether already acquired or
-hereafter acquired, that would be infringed by some manner, permitted
-by this License, of making, using, or selling its contributor version,
-but do not include claims that would be infringed only as a
-consequence of further modification of the contributor version. For
-purposes of this definition, "control" includes the right to grant
-patent sublicenses in a manner consistent with the requirements of
-this License.
-
- Each contributor grants you a non-exclusive, worldwide, royalty-free
-patent license under the contributor's essential patent claims, to
-make, use, sell, offer for sale, import and otherwise run, modify and
-propagate the contents of its contributor version.
-
- In the following three paragraphs, a "patent license" is any express
-agreement or commitment, however denominated, not to enforce a patent
-(such as an express permission to practice a patent or covenant not to
-sue for patent infringement). To "grant" such a patent license to a
-party means to make such an agreement or commitment not to enforce a
-patent against the party.
-
- If you convey a covered work, knowingly relying on a patent license,
-and the Corresponding Source of the work is not available for anyone
-to copy, free of charge and under the terms of this License, through a
-publicly available network server or other readily accessible means,
-then you must either (1) cause the Corresponding Source to be so
-available, or (2) arrange to deprive yourself of the benefit of the
-patent license for this particular work, or (3) arrange, in a manner
-consistent with the requirements of this License, to extend the patent
-license to downstream recipients. "Knowingly relying" means you have
-actual knowledge that, but for the patent license, your conveying the
-covered work in a country, or your recipient's use of the covered work
-in a country, would infringe one or more identifiable patents in that
-country that you have reason to believe are valid.
-
- If, pursuant to or in connection with a single transaction or
-arrangement, you convey, or propagate by procuring conveyance of, a
-covered work, and grant a patent license to some of the parties
-receiving the covered work authorizing them to use, propagate, modify
-or convey a specific copy of the covered work, then the patent license
-you grant is automatically extended to all recipients of the covered
-work and works based on it.
-
- A patent license is "discriminatory" if it does not include within
-the scope of its coverage, prohibits the exercise of, or is
-conditioned on the non-exercise of one or more of the rights that are
-specifically granted under this License. You may not convey a covered
-work if you are a party to an arrangement with a third party that is
-in the business of distributing software, under which you make payment
-to the third party based on the extent of your activity of conveying
-the work, and under which the third party grants, to any of the
-parties who would receive the covered work from you, a discriminatory
-patent license (a) in connection with copies of the covered work
-conveyed by you (or copies made from those copies), or (b) primarily
-for and in connection with specific products or compilations that
-contain the covered work, unless you entered into that arrangement,
-or that patent license was granted, prior to 28 March 2007.
-
- Nothing in this License shall be construed as excluding or limiting
-any implied license or other defenses to infringement that may
-otherwise be available to you under applicable patent law.
-
- 12. No Surrender of Others' Freedom.
-
- If conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License. If you cannot convey a
-covered work so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you may
-not convey it at all. For example, if you agree to terms that obligate you
-to collect a royalty for further conveying from those to whom you convey
-the Program, the only way you could satisfy both those terms and this
-License would be to refrain entirely from conveying the Program.
-
- 13. Use with the GNU Affero General Public License.
-
- Notwithstanding any other provision of this License, you have
-permission to link or combine any covered work with a work licensed
-under version 3 of the GNU Affero General Public License into a single
-combined work, and to convey the resulting work. The terms of this
-License will continue to apply to the part which is the covered work,
-but the special requirements of the GNU Affero General Public License,
-section 13, concerning interaction through a network will apply to the
-combination as such.
-
- 14. Revised Versions of this License.
-
- The Free Software Foundation may publish revised and/or new versions of
-the GNU General Public License from time to time. Such new versions will
-be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
- Each version is given a distinguishing version number. If the
-Program specifies that a certain numbered version of the GNU General
-Public License "or any later version" applies to it, you have the
-option of following the terms and conditions either of that numbered
-version or of any later version published by the Free Software
-Foundation. If the Program does not specify a version number of the
-GNU General Public License, you may choose any version ever published
-by the Free Software Foundation.
-
- If the Program specifies that a proxy can decide which future
-versions of the GNU General Public License can be used, that proxy's
-public statement of acceptance of a version permanently authorizes you
-to choose that version for the Program.
-
- Later license versions may give you additional or different
-permissions. However, no additional obligations are imposed on any
-author or copyright holder as a result of your choosing to follow a
-later version.
-
- 15. Disclaimer of Warranty.
-
- THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
-APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
-HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
-OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
-THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
-IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
-ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
- 16. Limitation of Liability.
-
- IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
-THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
-GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
-USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
-DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
-PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
-EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGES.
-
- 17. Interpretation of Sections 15 and 16.
-
- If the disclaimer of warranty and limitation of liability provided
-above cannot be given local legal effect according to their terms,
-reviewing courts shall apply local law that most closely approximates
-an absolute waiver of all civil liability in connection with the
-Program, unless a warranty or assumption of liability accompanies a
-copy of the Program in return for a fee.
-
- END OF TERMS AND CONDITIONS
-
- How to Apply These Terms to Your New Programs
-
- If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
- To do so, attach the following notices to the program. It is safest
-to attach them to the start of each source file to most effectively
-state the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
- <one line to give the program's name and a brief idea of what it does.>
- Copyright (C) <year> <name of author>
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Also add information on how to contact you by electronic and paper mail.
-
- If the program does terminal interaction, make it output a short
-notice like this when it starts in an interactive mode:
-
- <program> Copyright (C) <year> <name of author>
- This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
- This is free software, and you are welcome to redistribute it
- under certain conditions; type `show c' for details.
-
-The hypothetical commands `show w' and `show c' should show the appropriate
-parts of the General Public License. Of course, your program's commands
-might be different; for a GUI interface, you would use an "about box".
-
- You should also get your employer (if you work as a programmer) or school,
-if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU GPL, see
-<http://www.gnu.org/licenses/>.
-
- The GNU General Public License does not permit incorporating your program
-into proprietary programs. If your program is a subroutine library, you
-may consider it more useful to permit linking proprietary applications with
-the library. If this is what you want to do, use the GNU Lesser General
-Public License instead of this License. But first, please read
-<http://www.gnu.org/philosophy/why-not-lgpl.html>.
diff --git a/ext/kenlm/COPYING.LESSER.3 b/ext/kenlm/COPYING.LESSER.3
deleted file mode 100644
index cca7fc2..0000000
--- a/ext/kenlm/COPYING.LESSER.3
+++ /dev/null
@@ -1,165 +0,0 @@
- GNU LESSER GENERAL PUBLIC LICENSE
- Version 3, 29 June 2007
-
- Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-
- This version of the GNU Lesser General Public License incorporates
-the terms and conditions of version 3 of the GNU General Public
-License, supplemented by the additional permissions listed below.
-
- 0. Additional Definitions.
-
- As used herein, "this License" refers to version 3 of the GNU Lesser
-General Public License, and the "GNU GPL" refers to version 3 of the GNU
-General Public License.
-
- "The Library" refers to a covered work governed by this License,
-other than an Application or a Combined Work as defined below.
-
- An "Application" is any work that makes use of an interface provided
-by the Library, but which is not otherwise based on the Library.
-Defining a subclass of a class defined by the Library is deemed a mode
-of using an interface provided by the Library.
-
- A "Combined Work" is a work produced by combining or linking an
-Application with the Library. The particular version of the Library
-with which the Combined Work was made is also called the "Linked
-Version".
-
- The "Minimal Corresponding Source" for a Combined Work means the
-Corresponding Source for the Combined Work, excluding any source code
-for portions of the Combined Work that, considered in isolation, are
-based on the Application, and not on the Linked Version.
-
- The "Corresponding Application Code" for a Combined Work means the
-object code and/or source code for the Application, including any data
-and utility programs needed for reproducing the Combined Work from the
-Application, but excluding the System Libraries of the Combined Work.
-
- 1. Exception to Section 3 of the GNU GPL.
-
- You may convey a covered work under sections 3 and 4 of this License
-without being bound by section 3 of the GNU GPL.
-
- 2. Conveying Modified Versions.
-
- If you modify a copy of the Library, and, in your modifications, a
-facility refers to a function or data to be supplied by an Application
-that uses the facility (other than as an argument passed when the
-facility is invoked), then you may convey a copy of the modified
-version:
-
- a) under this License, provided that you make a good faith effort to
- ensure that, in the event an Application does not supply the
- function or data, the facility still operates, and performs
- whatever part of its purpose remains meaningful, or
-
- b) under the GNU GPL, with none of the additional permissions of
- this License applicable to that copy.
-
- 3. Object Code Incorporating Material from Library Header Files.
-
- The object code form of an Application may incorporate material from
-a header file that is part of the Library. You may convey such object
-code under terms of your choice, provided that, if the incorporated
-material is not limited to numerical parameters, data structure
-layouts and accessors, or small macros, inline functions and templates
-(ten or fewer lines in length), you do both of the following:
-
- a) Give prominent notice with each copy of the object code that the
- Library is used in it and that the Library and its use are
- covered by this License.
-
- b) Accompany the object code with a copy of the GNU GPL and this license
- document.
-
- 4. Combined Works.
-
- You may convey a Combined Work under terms of your choice that,
-taken together, effectively do not restrict modification of the
-portions of the Library contained in the Combined Work and reverse
-engineering for debugging such modifications, if you also do each of
-the following:
-
- a) Give prominent notice with each copy of the Combined Work that
- the Library is used in it and that the Library and its use are
- covered by this License.
-
- b) Accompany the Combined Work with a copy of the GNU GPL and this license
- document.
-
- c) For a Combined Work that displays copyright notices during
- execution, include the copyright notice for the Library among
- these notices, as well as a reference directing the user to the
- copies of the GNU GPL and this license document.
-
- d) Do one of the following:
-
- 0) Convey the Minimal Corresponding Source under the terms of this
- License, and the Corresponding Application Code in a form
- suitable for, and under terms that permit, the user to
- recombine or relink the Application with a modified version of
- the Linked Version to produce a modified Combined Work, in the
- manner specified by section 6 of the GNU GPL for conveying
- Corresponding Source.
-
- 1) Use a suitable shared library mechanism for linking with the
- Library. A suitable mechanism is one that (a) uses at run time
- a copy of the Library already present on the user's computer
- system, and (b) will operate properly with a modified version
- of the Library that is interface-compatible with the Linked
- Version.
-
- e) Provide Installation Information, but only if you would otherwise
- be required to provide such information under section 6 of the
- GNU GPL, and only to the extent that such information is
- necessary to install and execute a modified version of the
- Combined Work produced by recombining or relinking the
- Application with a modified version of the Linked Version. (If
- you use option 4d0, the Installation Information must accompany
- the Minimal Corresponding Source and Corresponding Application
- Code. If you use option 4d1, you must provide the Installation
- Information in the manner specified by section 6 of the GNU GPL
- for conveying Corresponding Source.)
-
- 5. Combined Libraries.
-
- You may place library facilities that are a work based on the
-Library side by side in a single library together with other library
-facilities that are not Applications and are not covered by this
-License, and convey such a combined library under terms of your
-choice, if you do both of the following:
-
- a) Accompany the combined library with a copy of the same work based
- on the Library, uncombined with any other library facilities,
- conveyed under the terms of this License.
-
- b) Give prominent notice with the combined library that part of it
- is a work based on the Library, and explaining where to find the
- accompanying uncombined form of the same work.
-
- 6. Revised Versions of the GNU Lesser General Public License.
-
- The Free Software Foundation may publish revised and/or new versions
-of the GNU Lesser General Public License from time to time. Such new
-versions will be similar in spirit to the present version, but may
-differ in detail to address new problems or concerns.
-
- Each version is given a distinguishing version number. If the
-Library as you received it specifies that a certain numbered version
-of the GNU Lesser General Public License "or any later version"
-applies to it, you have the option of following the terms and
-conditions either of that published version or of any later version
-published by the Free Software Foundation. If the Library as you
-received it does not specify a version number of the GNU Lesser
-General Public License, you may choose any version of the GNU Lesser
-General Public License ever published by the Free Software Foundation.
-
- If the Library as you received it specifies that a proxy can decide
-whether future versions of the GNU Lesser General Public License shall
-apply, that proxy's public statement of acceptance of any version is
-permanent authorization for you to choose that version for the
-Library.
diff --git a/ext/kenlm/Doxyfile b/ext/kenlm/Doxyfile
deleted file mode 100644
index 3abab65..0000000
--- a/ext/kenlm/Doxyfile
+++ /dev/null
@@ -1,1519 +0,0 @@
-# Doxyfile 1.6.1
-
-# This file describes the settings to be used by the documentation system
-# doxygen (www.doxygen.org) for a project
-#
-# All text after a hash (#) is considered a comment and will be ignored
-# The format is:
-# TAG = value [value, ...]
-# For lists items can also be appended using:
-# TAG += value [value, ...]
-# Values that contain spaces should be placed between quotes (" ")
-
-#---------------------------------------------------------------------------
-# Project related configuration options
-#---------------------------------------------------------------------------
-
-# This tag specifies the encoding used for all characters in the config file
-# that follow. The default is UTF-8 which is also the encoding used for all
-# text before the first occurrence of this tag. Doxygen uses libiconv (or the
-# iconv built into libc) for the transcoding. See
-# http://www.gnu.org/software/libiconv for the list of possible encodings.
-
-DOXYFILE_ENCODING = UTF-8
-
-# The PROJECT_NAME tag is a single word (or a sequence of words surrounded
-# by quotes) that should identify the project.
-
-PROJECT_NAME = KenLM
-
-# The PROJECT_NUMBER tag can be used to enter a project or revision number.
-# This could be handy for archiving the generated documentation or
-# if some version control system is used.
-
-PROJECT_NUMBER =
-
-# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute)
-# base path where the generated documentation will be put.
-# If a relative path is entered, it will be relative to the location
-# where doxygen was started. If left blank the current directory will be used.
-
-OUTPUT_DIRECTORY = doc
-
-# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create
-# 4096 sub-directories (in 2 levels) under the output directory of each output
-# format and will distribute the generated files over these directories.
-# Enabling this option can be useful when feeding doxygen a huge amount of
-# source files, where putting all generated files in the same directory would
-# otherwise cause performance problems for the file system.
-
-CREATE_SUBDIRS = NO
-
-# The OUTPUT_LANGUAGE tag is used to specify the language in which all
-# documentation generated by doxygen is written. Doxygen will use this
-# information to generate all constant output in the proper language.
-# The default language is English, other supported languages are:
-# Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional,
-# Croatian, Czech, Danish, Dutch, Esperanto, Farsi, Finnish, French, German,
-# Greek, Hungarian, Italian, Japanese, Japanese-en (Japanese with English
-# messages), Korean, Korean-en, Lithuanian, Norwegian, Macedonian, Persian,
-# Polish, Portuguese, Romanian, Russian, Serbian, Serbian-Cyrilic, Slovak,
-# Slovene, Spanish, Swedish, Ukrainian, and Vietnamese.
-
-OUTPUT_LANGUAGE = English
-
-# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will
-# include brief member descriptions after the members that are listed in
-# the file and class documentation (similar to JavaDoc).
-# Set to NO to disable this.
-
-BRIEF_MEMBER_DESC = YES
-
-# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend
-# the brief description of a member or function before the detailed description.
-# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the
-# brief descriptions will be completely suppressed.
-
-REPEAT_BRIEF = YES
-
-# This tag implements a quasi-intelligent brief description abbreviator
-# that is used to form the text in various listings. Each string
-# in this list, if found as the leading text of the brief description, will be
-# stripped from the text and the result after processing the whole list, is
-# used as the annotated text. Otherwise, the brief description is used as-is.
-# If left blank, the following values are used ("$name" is automatically
-# replaced with the name of the entity): "The $name class" "The $name widget"
-# "The $name file" "is" "provides" "specifies" "contains"
-# "represents" "a" "an" "the"
-
-ABBREVIATE_BRIEF =
-
-# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then
-# Doxygen will generate a detailed section even if there is only a brief
-# description.
-
-ALWAYS_DETAILED_SEC = YES
-
-# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all
-# inherited members of a class in the documentation of that class as if those
-# members were ordinary class members. Constructors, destructors and assignment
-# operators of the base classes will not be shown.
-
-INLINE_INHERITED_MEMB = NO
-
-# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full
-# path before files name in the file list and in the header files. If set
-# to NO the shortest path that makes the file name unique will be used.
-
-FULL_PATH_NAMES = YES
-
-# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag
-# can be used to strip a user-defined part of the path. Stripping is
-# only done if one of the specified strings matches the left-hand part of
-# the path. The tag can be used to show relative paths in the file list.
-# If left blank the directory from which doxygen is run is used as the
-# path to strip.
-
-STRIP_FROM_PATH =
-
-# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of
-# the path mentioned in the documentation of a class, which tells
-# the reader which header file to include in order to use a class.
-# If left blank only the name of the header file containing the class
-# definition is used. Otherwise one should specify the include paths that
-# are normally passed to the compiler using the -I flag.
-
-STRIP_FROM_INC_PATH =
-
-# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter
-# (but less readable) file names. This can be useful is your file systems
-# doesn't support long names like on DOS, Mac, or CD-ROM.
-
-SHORT_NAMES = NO
-
-# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen
-# will interpret the first line (until the first dot) of a JavaDoc-style
-# comment as the brief description. If set to NO, the JavaDoc
-# comments will behave just like regular Qt-style comments
-# (thus requiring an explicit @brief command for a brief description.)
-
-JAVADOC_AUTOBRIEF = YES
-
-# If the QT_AUTOBRIEF tag is set to YES then Doxygen will
-# interpret the first line (until the first dot) of a Qt-style
-# comment as the brief description. If set to NO, the comments
-# will behave just like regular Qt-style comments (thus requiring
-# an explicit \brief command for a brief description.)
-
-QT_AUTOBRIEF = NO
-
-# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen
-# treat a multi-line C++ special comment block (i.e. a block of //! or ///
-# comments) as a brief description. This used to be the default behaviour.
-# The new default is to treat a multi-line C++ comment block as a detailed
-# description. Set this tag to YES if you prefer the old behaviour instead.
-
-MULTILINE_CPP_IS_BRIEF = NO
-
-# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented
-# member inherits the documentation from any documented member that it
-# re-implements.
-
-INHERIT_DOCS = YES
-
-# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce
-# a new page for each member. If set to NO, the documentation of a member will
-# be part of the file/class/namespace that contains it.
-
-SEPARATE_MEMBER_PAGES = NO
-
-# The TAB_SIZE tag can be used to set the number of spaces in a tab.
-# Doxygen uses this value to replace tabs by spaces in code fragments.
-
-TAB_SIZE = 8
-
-# This tag can be used to specify a number of aliases that acts
-# as commands in the documentation. An alias has the form "name=value".
-# For example adding "sideeffect=\par Side Effects:\n" will allow you to
-# put the command \sideeffect (or @sideeffect) in the documentation, which
-# will result in a user-defined paragraph with heading "Side Effects:".
-# You can put \n's in the value part of an alias to insert newlines.
-
-ALIASES =
-
-# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C
-# sources only. Doxygen will then generate output that is more tailored for C.
-# For instance, some of the names that are used will be different. The list
-# of all members will be omitted, etc.
-
-OPTIMIZE_OUTPUT_FOR_C = NO
-
-# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java
-# sources only. Doxygen will then generate output that is more tailored for
-# Java. For instance, namespaces will be presented as packages, qualified
-# scopes will look different, etc.
-
-OPTIMIZE_OUTPUT_JAVA = NO
-
-# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran
-# sources only. Doxygen will then generate output that is more tailored for
-# Fortran.
-
-OPTIMIZE_FOR_FORTRAN = NO
-
-# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL
-# sources. Doxygen will then generate output that is tailored for
-# VHDL.
-
-OPTIMIZE_OUTPUT_VHDL = NO
-
-# Doxygen selects the parser to use depending on the extension of the files it parses.
-# With this tag you can assign which parser to use for a given extension.
-# Doxygen has a built-in mapping, but you can override or extend it using this tag.
-# The format is ext=language, where ext is a file extension, and language is one of
-# the parsers supported by doxygen: IDL, Java, Javascript, C#, C, C++, D, PHP,
-# Objective-C, Python, Fortran, VHDL, C, C++. For instance to make doxygen treat
-# .inc files as Fortran files (default is PHP), and .f files as C (default is Fortran),
-# use: inc=Fortran f=C. Note that for custom extensions you also need to set FILE_PATTERNS otherwise the files are not read by doxygen.
-
-EXTENSION_MAPPING =
-
-# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want
-# to include (a tag file for) the STL sources as input, then you should
-# set this tag to YES in order to let doxygen match functions declarations and
-# definitions whose arguments contain STL classes (e.g. func(std::string); v.s.
-# func(std::string) {}). This also make the inheritance and collaboration
-# diagrams that involve STL classes more complete and accurate.
-
-BUILTIN_STL_SUPPORT = NO
-
-# If you use Microsoft's C++/CLI language, you should set this option to YES to
-# enable parsing support.
-
-CPP_CLI_SUPPORT = NO
-
-# Set the SIP_SUPPORT tag to YES if your project consists of sip sources only.
-# Doxygen will parse them like normal C++ but will assume all classes use public
-# instead of private inheritance when no explicit protection keyword is present.
-
-SIP_SUPPORT = NO
-
-# For Microsoft's IDL there are propget and propput attributes to indicate getter
-# and setter methods for a property. Setting this option to YES (the default)
-# will make doxygen to replace the get and set methods by a property in the
-# documentation. This will only work if the methods are indeed getting or
-# setting a simple type. If this is not the case, or you want to show the
-# methods anyway, you should set this option to NO.
-
-IDL_PROPERTY_SUPPORT = YES
-
-# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC
-# tag is set to YES, then doxygen will reuse the documentation of the first
-# member in the group (if any) for the other members of the group. By default
-# all members of a group must be documented explicitly.
-
-DISTRIBUTE_GROUP_DOC = NO
-
-# Set the SUBGROUPING tag to YES (the default) to allow class member groups of
-# the same type (for instance a group of public functions) to be put as a
-# subgroup of that type (e.g. under the Public Functions section). Set it to
-# NO to prevent subgrouping. Alternatively, this can be done per class using
-# the \nosubgrouping command.
-
-SUBGROUPING = YES
-
-# When TYPEDEF_HIDES_STRUCT is enabled, a typedef of a struct, union, or enum
-# is documented as struct, union, or enum with the name of the typedef. So
-# typedef struct TypeS {} TypeT, will appear in the documentation as a struct
-# with name TypeT. When disabled the typedef will appear as a member of a file,
-# namespace, or class. And the struct will be named TypeS. This can typically
-# be useful for C code in case the coding convention dictates that all compound
-# types are typedef'ed and only the typedef is referenced, never the tag name.
-
-TYPEDEF_HIDES_STRUCT = NO
-
-# The SYMBOL_CACHE_SIZE determines the size of the internal cache use to
-# determine which symbols to keep in memory and which to flush to disk.
-# When the cache is full, less often used symbols will be written to disk.
-# For small to medium size projects (<1000 input files) the default value is
-# probably good enough. For larger projects a too small cache size can cause
-# doxygen to be busy swapping symbols to and from disk most of the time
-# causing a significant performance penality.
-# If the system has enough physical memory increasing the cache will improve the
-# performance by keeping more symbols in memory. Note that the value works on
-# a logarithmic scale so increasing the size by one will rougly double the
-# memory usage. The cache size is given by this formula:
-# 2^(16+SYMBOL_CACHE_SIZE). The valid range is 0..9, the default is 0,
-# corresponding to a cache size of 2^16 = 65536 symbols
-
-SYMBOL_CACHE_SIZE = 0
-
-#---------------------------------------------------------------------------
-# Build related configuration options
-#---------------------------------------------------------------------------
-
-# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in
-# documentation are documented, even if no documentation was available.
-# Private class members and static file members will be hidden unless
-# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES
-
-EXTRACT_ALL = NO
-
-# If the EXTRACT_PRIVATE tag is set to YES all private members of a class
-# will be included in the documentation.
-
-EXTRACT_PRIVATE = YES
-
-# If the EXTRACT_STATIC tag is set to YES all static members of a file
-# will be included in the documentation.
-
-EXTRACT_STATIC = NO
-
-# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs)
-# defined locally in source files will be included in the documentation.
-# If set to NO only classes defined in header files are included.
-
-EXTRACT_LOCAL_CLASSES = YES
-
-# This flag is only useful for Objective-C code. When set to YES local
-# methods, which are defined in the implementation section but not in
-# the interface are included in the documentation.
-# If set to NO (the default) only methods in the interface are included.
-
-EXTRACT_LOCAL_METHODS = NO
-
-# If this flag is set to YES, the members of anonymous namespaces will be
-# extracted and appear in the documentation as a namespace called
-# 'anonymous_namespace{file}', w
<TRUNCATED>