You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/04/19 21:34:06 UTC
[18/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/modules/order.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/modules/order.c b/ext/kenlm/jam-files/engine/modules/order.c
deleted file mode 100644
index 3a83d38..0000000
--- a/ext/kenlm/jam-files/engine/modules/order.c
+++ /dev/null
@@ -1,160 +0,0 @@
-/* Copyright 2004. Vladimir Prus
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#include "../lists.h"
-#include "../mem.h"
-#include "../native.h"
-#include "../object.h"
-#include "../strings.h"
-#include "../variable.h"
-
-
-/* Use quite klugy approach: when we add order dependency from 'a' to 'b', just
- * append 'b' to of value of variable 'a'.
- */
-LIST * add_pair( FRAME * frame, int flags )
-{
- LIST * arg = lol_get( frame->args, 0 );
- LISTITER iter = list_begin( arg );
- LISTITER const end = list_end( arg );
- var_set( frame->module, list_item( iter ), list_copy_range( arg, list_next(
- iter ), end ), VAR_APPEND );
- return L0;
-}
-
-
-/* Given a list and a value, returns position of that value in the list, or -1
- * if not found.
- */
-int list_index( LIST * list, OBJECT * value )
-{
- int result = 0;
- LISTITER iter = list_begin( list );
- LISTITER const end = list_end( list );
- for ( ; iter != end; iter = list_next( iter ), ++result )
- if ( object_equal( list_item( iter ), value ) )
- return result;
- return -1;
-}
-
-enum colors { white, gray, black };
-
-
-/* Main routine for topological sort. Calls itself recursively on all adjacent
- * vertices which were not yet visited. After that, 'current_vertex' is added to
- * '*result_ptr'.
- */
-void do_ts( int * * graph, int current_vertex, int * colors, int * * result_ptr
- )
-{
- int i;
-
- colors[ current_vertex ] = gray;
- for ( i = 0; graph[ current_vertex ][ i ] != -1; ++i )
- {
- int adjacent_vertex = graph[ current_vertex ][ i ];
- if ( colors[ adjacent_vertex ] == white )
- do_ts( graph, adjacent_vertex, colors, result_ptr );
- /* The vertex is either black, in which case we do not have to do
- * anything, or gray, in which case we have a loop. If we have a loop,
- * it is not clear what useful diagnostic we can emit, so we emit
- * nothing.
- */
- }
- colors[ current_vertex ] = black;
- **result_ptr = current_vertex;
- ( *result_ptr )++;
-}
-
-
-void topological_sort( int * * graph, int num_vertices, int * result )
-{
- int i;
- int * colors = ( int * )BJAM_CALLOC( num_vertices, sizeof( int ) );
- for ( i = 0; i < num_vertices; ++i )
- colors[ i ] = white;
-
- for ( i = 0; i < num_vertices; ++i )
- if ( colors[ i ] == white )
- do_ts( graph, i, colors, &result );
-
- BJAM_FREE( colors );
-}
-
-
-LIST * order( FRAME * frame, int flags )
-{
- LIST * arg = lol_get( frame->args, 0 );
- LIST * result = L0;
- int src;
- LISTITER iter = list_begin( arg );
- LISTITER const end = list_end( arg );
-
- /* We need to create a graph of order dependencies between the passed
- * objects. We assume there are no duplicates passed to 'add_pair'.
- */
- int length = list_length( arg );
- int * * graph = ( int * * )BJAM_CALLOC( length, sizeof( int * ) );
- int * order = ( int * )BJAM_MALLOC( ( length + 1 ) * sizeof( int ) );
-
- for ( src = 0; iter != end; iter = list_next( iter ), ++src )
- {
- /* For all objects this one depends upon, add elements to 'graph'. */
- LIST * dependencies = var_get( frame->module, list_item( iter ) );
- int index = 0;
- LISTITER dep_iter = list_begin( dependencies );
- LISTITER const dep_end = list_end( dependencies );
-
- graph[ src ] = ( int * )BJAM_CALLOC( list_length( dependencies ) + 1,
- sizeof( int ) );
- for ( ; dep_iter != dep_end; dep_iter = list_next( dep_iter ) )
- {
- int const dst = list_index( arg, list_item( dep_iter ) );
- if ( dst != -1 )
- graph[ src ][ index++ ] = dst;
- }
- graph[ src ][ index ] = -1;
- }
-
- topological_sort( graph, length, order );
-
- {
- int index = length - 1;
- for ( ; index >= 0; --index )
- {
- int i;
- LISTITER iter = list_begin( arg );
- LISTITER const end = list_end( arg );
- for ( i = 0; i < order[ index ]; ++i, iter = list_next( iter ) );
- result = list_push_back( result, object_copy( list_item( iter ) ) );
- }
- }
-
- /* Clean up */
- {
- int i;
- for ( i = 0; i < length; ++i )
- BJAM_FREE( graph[ i ] );
- BJAM_FREE( graph );
- BJAM_FREE( order );
- }
-
- return result;
-}
-
-
-void init_order()
-{
- {
- char const * args[] = { "first", "second", 0 };
- declare_native_rule( "class@order", "add-pair", args, add_pair, 1 );
- }
-
- {
- char const * args[] = { "objects", "*", 0 };
- declare_native_rule( "class@order", "order", args, order, 1 );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/modules/path.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/modules/path.c b/ext/kenlm/jam-files/engine/modules/path.c
deleted file mode 100644
index f8dedac..0000000
--- a/ext/kenlm/jam-files/engine/modules/path.c
+++ /dev/null
@@ -1,25 +0,0 @@
-/* Copyright Vladimir Prus 2003.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#include "../constants.h"
-#include "../frames.h"
-#include "../lists.h"
-#include "../native.h"
-#include "../timestamp.h"
-
-
-LIST * path_exists( FRAME * frame, int flags )
-{
- return file_query( list_front( lol_get( frame->args, 0 ) ) ) ?
- list_new( object_copy( constant_true ) ) : L0;
-}
-
-
-void init_path()
-{
- char const * args[] = { "location", 0 };
- declare_native_rule( "path", "exists", args, path_exists, 1 );
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/modules/property-set.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/modules/property-set.c b/ext/kenlm/jam-files/engine/modules/property-set.c
deleted file mode 100644
index 21e35d5..0000000
--- a/ext/kenlm/jam-files/engine/modules/property-set.c
+++ /dev/null
@@ -1,330 +0,0 @@
-/*
- * Copyright 2013 Steven Watanabe
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#include "../object.h"
-#include "../lists.h"
-#include "../modules.h"
-#include "../rules.h"
-#include "../variable.h"
-#include "../native.h"
-#include "../compile.h"
-#include "../mem.h"
-#include "../constants.h"
-#include "string.h"
-
-struct ps_map_entry
-{
- struct ps_map_entry * next;
- LIST * key;
- OBJECT * value;
-};
-
-struct ps_map
-{
- struct ps_map_entry * * table;
- size_t table_size;
- size_t num_elems;
-};
-
-static unsigned list_hash(LIST * key)
-{
- unsigned int hash = 0;
- LISTITER iter = list_begin( key ), end = list_end( key );
- for ( ; iter != end; ++iter )
- {
- hash = hash * 2147059363 + object_hash( list_item( iter ) );
- }
- return hash;
-}
-
-static int list_equal( LIST * lhs, LIST * rhs )
-{
- LISTITER lhs_iter, lhs_end, rhs_iter;
- if ( list_length( lhs ) != list_length( rhs ) )
- {
- return 0;
- }
- lhs_iter = list_begin( lhs );
- lhs_end = list_end( lhs );
- rhs_iter = list_begin( rhs );
- for ( ; lhs_iter != lhs_end; ++lhs_iter, ++rhs_iter )
- {
- if ( ! object_equal( list_item( lhs_iter ), list_item( rhs_iter ) ) )
- {
- return 0;
- }
- }
- return 1;
-}
-
-static void ps_map_init( struct ps_map * map )
-{
- size_t i;
- map->table_size = 2;
- map->num_elems = 0;
- map->table = BJAM_MALLOC( map->table_size * sizeof( struct ps_map_entry * ) );
- for ( i = 0; i < map->table_size; ++i )
- {
- map->table[ i ] = NULL;
- }
-}
-
-static void ps_map_destroy( struct ps_map * map )
-{
- size_t i;
- for ( i = 0; i < map->table_size; ++i )
- {
- struct ps_map_entry * pos;
- for ( pos = map->table[ i ]; pos; )
- {
- struct ps_map_entry * tmp = pos->next;
- BJAM_FREE( pos );
- pos = tmp;
- }
- }
- BJAM_FREE( map->table );
-}
-
-static void ps_map_rehash( struct ps_map * map )
-{
- struct ps_map old = *map;
- size_t i;
- map->table = BJAM_MALLOC( map->table_size * 2 * sizeof( struct ps_map_entry * ) );
- map->table_size *= 2;
- for ( i = 0; i < map->table_size; ++i )
- {
- map->table[ i ] = NULL;
- }
- for ( i = 0; i < old.table_size; ++i )
- {
- struct ps_map_entry * pos;
- for ( pos = old.table[ i ]; pos; )
- {
- struct ps_map_entry * tmp = pos->next;
-
- unsigned hash_val = list_hash( pos->key );
- unsigned bucket = hash_val % map->table_size;
- pos->next = map->table[ bucket ];
- map->table[ bucket ] = pos;
-
- pos = tmp;
- }
- }
- BJAM_FREE( old.table );
-}
-
-static struct ps_map_entry * ps_map_insert(struct ps_map * map, LIST * key)
-{
- unsigned hash_val = list_hash( key );
- unsigned bucket = hash_val % map->table_size;
- struct ps_map_entry * pos;
- for ( pos = map->table[bucket]; pos ; pos = pos->next )
- {
- if ( list_equal( pos->key, key ) )
- return pos;
- }
-
- if ( map->num_elems >= map->table_size )
- {
- ps_map_rehash( map );
- bucket = hash_val % map->table_size;
- }
- pos = BJAM_MALLOC( sizeof( struct ps_map_entry ) );
- pos->next = map->table[bucket];
- pos->key = key;
- pos->value = 0;
- map->table[bucket] = pos;
- ++map->num_elems;
- return pos;
-}
-
-static struct ps_map all_property_sets;
-
-LIST * property_set_create( FRAME * frame, int flags )
-{
- LIST * properties = lol_get( frame->args, 0 );
- LIST * sorted = list_sort( properties );
- LIST * unique = list_unique( sorted );
- struct ps_map_entry * pos = ps_map_insert( &all_property_sets, unique );
- list_free( sorted );
- if ( pos->value )
- {
- list_free( unique );
- return list_new( object_copy( pos->value ) );
- }
- else
- {
- OBJECT * rulename = object_new( "new" );
- OBJECT * varname = object_new( "self.raw" );
- LIST * val = call_rule( rulename, frame,
- list_new( object_new( "property-set" ) ), 0 );
- LISTITER iter, end;
- object_free( rulename );
- pos->value = list_front( val );
- var_set( bindmodule( pos->value ), varname, unique, VAR_SET );
- object_free( varname );
-
- for ( iter = list_begin( unique ), end = list_end( unique ); iter != end; ++iter )
- {
- const char * str = object_str( list_item( iter ) );
- if ( str[ 0 ] != '<' || ! strchr( str, '>' ) )
- {
- string message[ 1 ];
- string_new( message );
- string_append( message, "Invalid property: '" );
- string_append( message, str );
- string_append( message, "'" );
- rulename = object_new( "errors.error" );
- call_rule( rulename, frame,
- list_new( object_new( message->value ) ), 0 );
- /* unreachable */
- string_free( message );
- object_free( rulename );
- }
- }
-
- return val;
- }
-}
-
-/* binary search for the property value */
-LIST * property_set_get( FRAME * frame, int flags )
-{
- OBJECT * varname = object_new( "self.raw" );
- LIST * props = var_get( frame->module, varname );
- const char * name = object_str( list_front( lol_get( frame->args, 0 ) ) );
- size_t name_len = strlen( name );
- LISTITER begin, end;
- LIST * result = L0;
- object_free( varname );
-
- /* Assumes random access */
- begin = list_begin( props ), end = list_end( props );
-
- while ( 1 )
- {
- ptrdiff_t diff = (end - begin);
- LISTITER mid = begin + diff / 2;
- int res;
- if ( diff == 0 )
- {
- return L0;
- }
- res = strncmp( object_str( list_item( mid ) ), name, name_len );
- if ( res < 0 )
- {
- begin = mid + 1;
- }
- else if ( res > 0 )
- {
- end = mid;
- }
- else /* We've found the property */
- {
- /* Find the beginning of the group */
- LISTITER tmp = mid;
- while ( tmp > begin )
- {
- --tmp;
- res = strncmp( object_str( list_item( tmp ) ), name, name_len );
- if ( res != 0 )
- {
- ++tmp;
- break;
- }
- }
- begin = tmp;
- /* Find the end of the group */
- tmp = mid + 1;
- while ( tmp < end )
- {
- res = strncmp( object_str( list_item( tmp ) ), name, name_len );
- if ( res != 0 ) break;
- ++tmp;
- }
- end = tmp;
- break;
- }
- }
-
- for ( ; begin != end; ++begin )
- {
- result = list_push_back( result,
- object_new( object_str( list_item( begin ) ) + name_len ) );
- }
-
- return result;
-}
-
-/* binary search for the property value */
-LIST * property_set_contains_features( FRAME * frame, int flags )
-{
- OBJECT * varname = object_new( "self.raw" );
- LIST * props = var_get( frame->module, varname );
- LIST * features = lol_get( frame->args, 0 );
- LIST * result = L0;
- LISTITER features_iter = list_begin( features );
- LISTITER features_end = list_end( features ) ;
- object_free( varname );
-
- for ( ; features_iter != features_end; ++features_iter )
- {
- const char * name = object_str( list_item( features_iter ) );
- size_t name_len = strlen( name );
- LISTITER begin, end;
- /* Assumes random access */
- begin = list_begin( props ), end = list_end( props );
-
- while ( 1 )
- {
- ptrdiff_t diff = (end - begin);
- LISTITER mid = begin + diff / 2;
- int res;
- if ( diff == 0 )
- {
- /* The feature is missing */
- return L0;
- }
- res = strncmp( object_str( list_item( mid ) ), name, name_len );
- if ( res < 0 )
- {
- begin = mid + 1;
- }
- else if ( res > 0 )
- {
- end = mid;
- }
- else /* We've found the property */
- {
- break;
- }
- }
- }
- return list_new( object_copy( constant_true ) );
-}
-
-void init_property_set()
-{
- {
- char const * args[] = { "raw-properties", "*", 0 };
- declare_native_rule( "property-set", "create", args, property_set_create, 1 );
- }
- {
- char const * args[] = { "feature", 0 };
- declare_native_rule( "class@property-set", "get", args, property_set_get, 1 );
- }
- {
- char const * args[] = { "features", "*", 0 };
- declare_native_rule( "class@property-set", "contains-features", args, property_set_contains_features, 1 );
- }
- ps_map_init( &all_property_sets );
-}
-
-void property_set_done()
-{
- ps_map_destroy( &all_property_sets );
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/modules/readme.txt
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/modules/readme.txt b/ext/kenlm/jam-files/engine/modules/readme.txt
deleted file mode 100644
index 2edf6e1..0000000
--- a/ext/kenlm/jam-files/engine/modules/readme.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-
-This directory constains sources which declare native
-rules for Boost.Build modules.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/modules/regex.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/modules/regex.c b/ext/kenlm/jam-files/engine/modules/regex.c
deleted file mode 100644
index d9f8177..0000000
--- a/ext/kenlm/jam-files/engine/modules/regex.c
+++ /dev/null
@@ -1,220 +0,0 @@
-/*
- * Copyright 2003. Vladimir Prus
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#include "../mem.h"
-#include "../native.h"
-#include "../strings.h"
-#include "../subst.h"
-
-/*
-rule split ( string separator )
-{
- local result ;
- local s = $(string) ;
-
- local match = 1 ;
- while $(match)
- {
- match = [ MATCH ^(.*)($(separator))(.*) : $(s) ] ;
- if $(match)
- {
- match += "" ; # in case 3rd item was empty - works around MATCH bug
- result = $(match[3]) $(result) ;
- s = $(match[1]) ;
- }
- }
- return $(s) $(result) ;
-}
-*/
-
-LIST * regex_split( FRAME * frame, int flags )
-{
- LIST * args = lol_get( frame->args, 0 );
- OBJECT * s;
- OBJECT * separator;
- regexp * re;
- const char * pos;
- LIST * result = L0;
- LISTITER iter = list_begin( args );
- s = list_item( iter );
- separator = list_item( list_next( iter ) );
-
- re = regex_compile( separator );
-
- pos = object_str( s );
- while ( regexec( re, pos ) )
- {
- result = list_push_back( result, object_new_range( pos, re->startp[ 0 ] - pos ) );
- pos = re->endp[ 0 ];
- }
-
- result = list_push_back( result, object_new( pos ) );
-
- return result;
-}
-
-/*
-rule replace (
- string # The string to modify.
- match # The characters to replace.
- replacement # The string to replace with.
- )
-{
- local result = "" ;
- local parts = 1 ;
- while $(parts)
- {
- parts = [ MATCH ^(.*)($(match))(.*) : $(string) ] ;
- if $(parts)
- {
- parts += "" ;
- result = "$(replacement)$(parts[3])$(result)" ;
- string = $(parts[1]) ;
- }
- }
- string ?= "" ;
- result = "$(string)$(result)" ;
- return $(result) ;
-}
-*/
-
-LIST * regex_replace( FRAME * frame, int flags )
-{
- LIST * args = lol_get( frame->args, 0 );
- OBJECT * s;
- OBJECT * match;
- OBJECT * replacement;
- regexp * re;
- const char * pos;
- string buf[ 1 ];
- LIST * result;
- LISTITER iter = list_begin( args );
- s = list_item( iter );
- iter = list_next( iter );
- match = list_item( iter );
- iter = list_next( iter );
- replacement = list_item(iter );
-
- re = regex_compile( match );
-
- string_new( buf );
-
- pos = object_str( s );
- while ( regexec( re, pos ) )
- {
- string_append_range( buf, pos, re->startp[ 0 ] );
- string_append( buf, object_str( replacement ) );
- pos = re->endp[ 0 ];
- }
- string_append( buf, pos );
-
- result = list_new( object_new( buf->value ) );
-
- string_free( buf );
-
- return result;
-}
-
-/*
-rule transform ( list * : pattern : indices * )
-{
- indices ?= 1 ;
- local result ;
- for local e in $(list)
- {
- local m = [ MATCH $(pattern) : $(e) ] ;
- if $(m)
- {
- result += $(m[$(indices)]) ;
- }
- }
- return $(result) ;
-}
-*/
-
-LIST * regex_transform( FRAME * frame, int flags )
-{
- LIST * const l = lol_get( frame->args, 0 );
- LIST * const pattern = lol_get( frame->args, 1 );
- LIST * const indices_list = lol_get( frame->args, 2 );
- int * indices = 0;
- int size;
- LIST * result = L0;
-
- if ( !list_empty( indices_list ) )
- {
- int * p;
- LISTITER iter = list_begin( indices_list );
- LISTITER const end = list_end( indices_list );
- size = list_length( indices_list );
- indices = (int *)BJAM_MALLOC( size * sizeof( int ) );
- for ( p = indices; iter != end; iter = list_next( iter ) )
- *p++ = atoi( object_str( list_item( iter ) ) );
- }
- else
- {
- size = 1;
- indices = (int *)BJAM_MALLOC( sizeof( int ) );
- *indices = 1;
- }
-
- {
- /* Result is cached and intentionally never freed */
- regexp * const re = regex_compile( list_front( pattern ) );
-
- LISTITER iter = list_begin( l );
- LISTITER const end = list_end( l );
-
- string buf[ 1 ];
- string_new( buf );
-
- for ( ; iter != end; iter = list_next( iter ) )
- {
- if ( regexec( re, object_str( list_item( iter ) ) ) )
- {
- int i = 0;
- for ( ; i < size; ++i )
- {
- int const index = indices[ i ];
- /* Skip empty submatches. Not sure it is right in all cases,
- * but surely is right for the case for which this routine
- * is optimized -- header scanning.
- */
- if ( re->startp[ index ] != re->endp[ index ] )
- {
- string_append_range( buf, re->startp[ index ],
- re->endp[ index ] );
- result = list_push_back( result, object_new( buf->value
- ) );
- string_truncate( buf, 0 );
- }
- }
- }
- }
- string_free( buf );
- }
-
- BJAM_FREE( indices );
- return result;
-}
-
-
-void init_regex()
-{
- {
- char const * args[] = { "string", "separator", 0 };
- declare_native_rule( "regex", "split", args, regex_split, 1 );
- }
- {
- char const * args[] = { "string", "match", "replacement", 0 };
- declare_native_rule( "regex", "replace", args, regex_replace, 1 );
- }
- {
- char const * args[] = { "list", "*", ":", "pattern", ":", "indices", "*", 0 };
- declare_native_rule( "regex", "transform", args, regex_transform, 2 );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/modules/sequence.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/modules/sequence.c b/ext/kenlm/jam-files/engine/modules/sequence.c
deleted file mode 100644
index 08ed305..0000000
--- a/ext/kenlm/jam-files/engine/modules/sequence.c
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Copyright Vladimir Prus 2003.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#include "../native.h"
-#include "../object.h"
-#include "../lists.h"
-#include "../compile.h"
-
-#include <stdlib.h>
-
-
-#ifndef max
-# define max(a,b) ((a)>(b)?(a):(b))
-#endif
-
-
-LIST * sequence_select_highest_ranked( FRAME * frame, int flags )
-{
- /* Returns all of 'elements' for which corresponding element in parallel */
- /* list 'rank' is equal to the maximum value in 'rank'. */
-
- LIST * const elements = lol_get( frame->args, 0 );
- LIST * const rank = lol_get( frame->args, 1 );
-
- LIST * result = L0;
- int highest_rank = -1;
-
- {
- LISTITER iter = list_begin( rank );
- LISTITER const end = list_end( rank );
- for ( ; iter != end; iter = list_next( iter ) )
- {
- int const current = atoi( object_str( list_item( iter ) ) );
- highest_rank = max( highest_rank, current );
- }
- }
-
- {
- LISTITER iter = list_begin( rank );
- LISTITER const end = list_end( rank );
- LISTITER elements_iter = list_begin( elements );
- LISTITER const elements_end = list_end( elements );
- for ( ; iter != end; iter = list_next( iter ), elements_iter =
- list_next( elements_iter ) )
- if ( atoi( object_str( list_item( iter ) ) ) == highest_rank )
- result = list_push_back( result, object_copy( list_item(
- elements_iter ) ) );
- }
-
- return result;
-}
-
-LIST * sequence_transform( FRAME * frame, int flags )
-{
- LIST * function = lol_get( frame->args, 0 );
- LIST * sequence = lol_get( frame->args, 1 );
- LIST * result = L0;
- OBJECT * function_name = list_front( function );
- LISTITER args_begin = list_next( list_begin( function ) ), args_end = list_end( function );
- LISTITER iter = list_begin( sequence ), end = list_end( sequence );
- RULE * rule = bindrule( function_name, frame->prev->module );
-
- for ( ; iter != end; iter = list_next( iter ) )
- {
- FRAME inner[ 1 ];
-
- frame_init( inner );
- inner->prev = frame;
- inner->prev_user = frame->prev_user;
- inner->module = frame->prev->module;
-
- lol_add( inner->args, list_push_back( list_copy_range( function, args_begin, args_end ), object_copy( list_item( iter ) ) ) );
- result = list_append( result, evaluate_rule( rule, function_name, inner ) );
-
- frame_free( inner );
- }
-
- return result;
-}
-
-void init_sequence()
-{
- {
- char const * args[] = { "elements", "*", ":", "rank", "*", 0 };
- declare_native_rule( "sequence", "select-highest-ranked", args,
- sequence_select_highest_ranked, 1 );
- }
- {
- char const * args[] = { "function", "+", ":", "sequence", "*", 0 };
- declare_native_rule( "sequence", "transform", args,
- sequence_transform, 1 );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/modules/set.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/modules/set.c b/ext/kenlm/jam-files/engine/modules/set.c
deleted file mode 100644
index 77a314d..0000000
--- a/ext/kenlm/jam-files/engine/modules/set.c
+++ /dev/null
@@ -1,43 +0,0 @@
-/* Copyright Vladimir Prus 2003. Distributed under the Boost */
-/* Software License, Version 1.0. (See accompanying */
-/* file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) */
-
-#include "../native.h"
-#include "../object.h"
-
-/*
- local result = ;
- local element ;
- for element in $(B)
- {
- if ! ( $(element) in $(A) )
- {
- result += $(element) ;
- }
- }
- return $(result) ;
-*/
-LIST *set_difference( FRAME *frame, int flags )
-{
-
- LIST* b = lol_get( frame->args, 0 );
- LIST* a = lol_get( frame->args, 1 );
-
- LIST* result = L0;
- LISTITER iter = list_begin( b ), end = list_end( b );
- for( ; iter != end; iter = list_next( iter ) )
- {
- if (!list_in(a, list_item(iter)))
- result = list_push_back(result, object_copy(list_item(iter)));
- }
- return result;
-}
-
-void init_set()
-{
- {
- const char* args[] = { "B", "*", ":", "A", "*", 0 };
- declare_native_rule("set", "difference", args, set_difference, 1);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/native.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/native.c b/ext/kenlm/jam-files/engine/native.c
deleted file mode 100644
index 68828aa..0000000
--- a/ext/kenlm/jam-files/engine/native.c
+++ /dev/null
@@ -1,34 +0,0 @@
-/* Copyright 2003. Vladimir Prus
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#include "native.h"
-
-#include "hash.h"
-
-#include <assert.h>
-
-
-void declare_native_rule( char const * module, char const * rule,
- char const * * args, LIST * (*f)( FRAME *, int ), int version )
-{
- OBJECT * const module_obj = module ? object_new( module ) : 0 ;
- module_t * m = bindmodule( module_obj );
- if ( module_obj )
- object_free( module_obj );
- if ( !m->native_rules )
- m->native_rules = hashinit( sizeof( native_rule_t ), "native rules" );
-
- {
- OBJECT * const name = object_new( rule );
- int found;
- native_rule_t * const np = (native_rule_t *)hash_insert(
- m->native_rules, name, &found );
- np->name = name;
- assert( !found );
- np->procedure = function_builtin( f, 0, args );
- np->version = version;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/native.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/native.h b/ext/kenlm/jam-files/engine/native.h
deleted file mode 100644
index 6d38d01..0000000
--- a/ext/kenlm/jam-files/engine/native.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/* Copyright 2003. David Abrahams
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#ifndef NATIVE_H_VP_2003_12_09
-#define NATIVE_H_VP_2003_12_09
-
-#include "function.h"
-#include "frames.h"
-#include "lists.h"
-#include "object.h"
-
-typedef struct native_rule_t
-{
- OBJECT * name;
- FUNCTION * procedure;
-
- /* Version of the interface that the native rule provides. It is possible
- * that we want to change the set parameter for existing native rule. In
- * that case, version number should be incremented so Boost.Build can check
- * for the version it relies on.
- *
- * Versions are numbered from 1.
- */
- int version;
-} native_rule_t;
-/* MSVC debugger gets confused unless the native_rule_t typedef is provided. */
-
-void declare_native_rule( char const * module, char const * rule,
- char const * * args, LIST * (*f)( FRAME *, int ), int version );
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/object.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/object.c b/ext/kenlm/jam-files/engine/object.c
deleted file mode 100644
index ef46e4a..0000000
--- a/ext/kenlm/jam-files/engine/object.c
+++ /dev/null
@@ -1,394 +0,0 @@
-/*
- * Copyright 1993, 1995 Christopher Seiwald.
- * Copyright 2011 Steven Watanabe
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/*
- * object.c - object manipulation routines
- *
- * External functions:
- * object_new() - create an object from a string
- * object_new_range() - create an object from a string of given length
- * object_copy() - return a copy of an object
- * object_free() - free an object
- * object_str() - get the string value of an object
- * object_done() - free string tables
- *
- * This implementation builds a hash table of all strings, so that multiple
- * calls of object_new() on the same string allocate memory for the string once.
- * Strings are never actually freed.
- */
-
-#include "jam.h"
-#include "object.h"
-
-#include <assert.h>
-#include <stddef.h>
-#include <stdlib.h>
-
-
-#define OBJECT_MAGIC 0xa762e0e3u
-
-#ifndef object_copy
-
-struct hash_header
-{
-#ifndef NDEBUG
- unsigned int magic;
-#endif
- unsigned int hash;
- struct hash_item * next;
-};
-
-#endif
-
-struct hash_item
-{
- struct hash_header header;
- char data[ 1 ];
-};
-
-#define ALLOC_ALIGNMENT (sizeof(struct hash_item) - sizeof(struct hash_header))
-
-typedef struct string_set
-{
- unsigned int num;
- unsigned int size;
- struct hash_item * * data;
-} string_set;
-
-static string_set strhash;
-static int strtotal = 0;
-static int strcount_in = 0;
-static int strcount_out = 0;
-
-
-/*
- * Immortal string allocator implementation speeds string allocation and cuts
- * down on internal fragmentation.
- */
-
-#define STRING_BLOCK 4096
-typedef struct strblock
-{
- struct strblock * next;
- char data[ STRING_BLOCK ];
-} strblock;
-
-static strblock * strblock_chain = 0;
-
-/* Storage remaining in the current strblock */
-static char * storage_start = 0;
-static char * storage_finish = 0;
-
-
-/*
- * allocate() - Allocate n bytes of immortal string storage.
- */
-
-static char * allocate( size_t n )
-{
-#ifdef BJAM_NEWSTR_NO_ALLOCATE
- return (char *)BJAM_MALLOC( n );
-#else
- /* See if we can grab storage from an existing block. */
- size_t remaining = storage_finish - storage_start;
- n = ( ( n + ALLOC_ALIGNMENT - 1 ) / ALLOC_ALIGNMENT ) * ALLOC_ALIGNMENT;
- if ( remaining >= n )
- {
- char * result = storage_start;
- storage_start += n;
- return result;
- }
- else /* Must allocate a new block. */
- {
- strblock * new_block;
- size_t nalloc = n;
- if ( nalloc < STRING_BLOCK )
- nalloc = STRING_BLOCK;
-
- /* Allocate a new block and link into the chain. */
- new_block = (strblock *)BJAM_MALLOC( offsetof( strblock, data[ 0 ] ) +
- nalloc * sizeof( new_block->data[ 0 ] ) );
- if ( new_block == 0 )
- return 0;
- new_block->next = strblock_chain;
- strblock_chain = new_block;
-
- /* Take future allocations out of the larger remaining space. */
- if ( remaining < nalloc - n )
- {
- storage_start = new_block->data + n;
- storage_finish = new_block->data + nalloc;
- }
- return new_block->data;
- }
-#endif
-}
-
-
-static unsigned int hash_keyval( char const * key, int const size )
-{
- unsigned int const magic = 2147059363;
- unsigned int hash = 0;
-
- unsigned int i;
- for ( i = 0; i < size / sizeof( unsigned int ); ++i )
- {
- unsigned int val;
- memcpy( &val, key, sizeof( unsigned int ) );
- hash = hash * magic + val;
- key += sizeof( unsigned int );
- }
-
- {
- unsigned int val = 0;
- memcpy( &val, key, size % sizeof( unsigned int ) );
- hash = hash * magic + val;
- }
-
- return hash + ( hash >> 17 );
-}
-
-
-static void string_set_init( string_set * set )
-{
- set->size = 0;
- set->num = 4;
- set->data = (struct hash_item * *)BJAM_MALLOC( set->num * sizeof( struct hash_item * ) );
- memset( set->data, 0, set->num * sizeof( struct hash_item * ) );
-}
-
-
-static void string_set_done( string_set * set )
-{
- BJAM_FREE( set->data );
-}
-
-
-static void string_set_resize( string_set * set )
-{
- unsigned i;
- string_set new_set;
- new_set.num = set->num * 2;
- new_set.size = set->size;
- new_set.data = (struct hash_item * *)BJAM_MALLOC( sizeof( struct hash_item *
- ) * new_set.num );
- memset( new_set.data, 0, sizeof( struct hash_item * ) * new_set.num );
- for ( i = 0; i < set->num; ++i )
- {
- while ( set->data[ i ] )
- {
- struct hash_item * temp = set->data[ i ];
- unsigned pos = temp->header.hash % new_set.num;
- set->data[ i ] = temp->header.next;
- temp->header.next = new_set.data[ pos ];
- new_set.data[ pos ] = temp;
- }
- }
- BJAM_FREE( set->data );
- *set = new_set;
-}
-
-
-static char const * string_set_insert( string_set * set, char const * string,
- int const size )
-{
- unsigned hash = hash_keyval( string, size );
- unsigned pos = hash % set->num;
-
- struct hash_item * result;
-
- for ( result = set->data[ pos ]; result; result = result->header.next )
- if ( !strncmp( result->data, string, size ) && !result->data[ size ] )
- return result->data;
-
- if ( set->size >= set->num )
- {
- string_set_resize( set );
- pos = hash % set->num;
- }
-
- result = (struct hash_item *)allocate( sizeof( struct hash_header ) + size +
- 1 );
- result->header.hash = hash;
- result->header.next = set->data[ pos ];
-#ifndef NDEBUG
- result->header.magic = OBJECT_MAGIC;
-#endif
- memcpy( result->data, string, size );
- result->data[ size ] = '\0';
- assert( hash_keyval( result->data, size ) == result->header.hash );
- set->data[ pos ] = result;
- strtotal += size + 1;
- ++set->size;
-
- return result->data;
-}
-
-
-static struct hash_item * object_get_item( OBJECT * obj )
-{
- return (struct hash_item *)( (char *)obj - offsetof( struct hash_item, data
- ) );
-}
-
-
-static void object_validate( OBJECT * obj )
-{
- assert( obj );
- assert( object_get_item( obj )->header.magic == OBJECT_MAGIC );
-}
-
-
-/*
- * object_new_range() - create an object from a string of given length
- */
-
-OBJECT * object_new_range( char const * const string, int const size )
-{
- ++strcount_in;
-
-#ifdef BJAM_NO_MEM_CACHE
- {
- struct hash_item * const m = (struct hash_item *)BJAM_MALLOC( sizeof(
- struct hash_header ) + size + 1 );
- strtotal += size + 1;
- memcpy( m->data, string, size );
- m->data[ size ] = '\0';
- m->header.magic = OBJECT_MAGIC;
- return (OBJECT *)m->data;
- }
-#else
- if ( !strhash.data )
- string_set_init( &strhash );
- return (OBJECT *)string_set_insert( &strhash, string, size );
-#endif
-}
-
-
-/*
- * object_new() - create an object from a string
- */
-
-OBJECT * object_new( char const * const string )
-{
- return object_new_range( string, strlen( string ) );
-}
-
-
-#ifndef object_copy
-
-/*
- * object_copy() - return a copy of an object
- */
-
-OBJECT * object_copy( OBJECT * obj )
-{
- object_validate( obj );
-#ifdef BJAM_NO_MEM_CACHE
- return object_new( object_str( obj ) );
-#else
- ++strcount_in;
- return obj;
-#endif
-}
-
-
-/*
- * object_free() - free an object
- */
-
-void object_free( OBJECT * obj )
-{
- object_validate( obj );
-#ifdef BJAM_NO_MEM_CACHE
- BJAM_FREE( object_get_item( obj ) );
-#endif
- ++strcount_out;
-}
-
-
-/*
- * object_str() - return the OBJECT's internal C string
- */
-
-char const * object_str( OBJECT * obj )
-{
- object_validate( obj );
- return (char const *)obj;
-}
-
-
-/*
- * object_equal() - compare two objects
- */
-
-int object_equal( OBJECT * lhs, OBJECT * rhs )
-{
- object_validate( lhs );
- object_validate( rhs );
-#ifdef BJAM_NO_MEM_CACHE
- return !strcmp( object_str( lhs ), object_str( rhs ) );
-#else
- assert( ( lhs == rhs ) == !strcmp( object_str( lhs ), object_str( rhs ) ) );
- return lhs == rhs;
-#endif
-}
-
-
-/*
- * object_hash() - returns the hash value of an object
- */
-
-unsigned int object_hash( OBJECT * obj )
-{
- object_validate( obj );
-#ifdef BJAM_NO_MEM_CACHE
- return hash_keyval( object_str( obj ), strlen( object_str( obj ) ) );
-#else
- return object_get_item( obj )->header.hash;
-#endif
-}
-
-#endif
-
-/*
- * object_done() - free string tables.
- */
-
-void object_done()
-{
-#ifdef BJAM_NEWSTR_NO_ALLOCATE
- unsigned i;
- for ( i = 0; i < strhash.num; ++i )
- {
- while ( strhash.data[ i ] )
- {
- struct hash_item * item = strhash.data[ i ];
- strhash.data[ i ] = item->header.next;
- BJAM_FREE( item );
- }
- }
-#else
- /* Reclaim string blocks. */
- while ( strblock_chain )
- {
- strblock * const n = strblock_chain->next;
- BJAM_FREE( strblock_chain );
- strblock_chain = n;
- }
-#endif
-
- string_set_done( &strhash );
-
- if ( DEBUG_MEM )
- {
- printf( "%dK in strings\n", strtotal / 1024 );
- if ( strcount_in != strcount_out )
- printf( "--- %d strings of %d dangling\n", strcount_in -
- strcount_out, strcount_in );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/object.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/object.h b/ext/kenlm/jam-files/engine/object.h
deleted file mode 100644
index cabb9f6..0000000
--- a/ext/kenlm/jam-files/engine/object.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright 2011 Steven Watanabe
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/*
- * object.h - object manipulation routines
- */
-
-#ifndef BOOST_JAM_OBJECT_H
-#define BOOST_JAM_OBJECT_H
-
-typedef struct _object OBJECT;
-
-OBJECT * object_new( char const * const );
-OBJECT * object_new_range( char const * const, int const size );
-void object_done( void );
-
-#if defined(NDEBUG) && !defined(BJAM_NO_MEM_CACHE)
-
-struct hash_header
-{
- unsigned int hash;
- struct hash_item * next;
-};
-
-#define object_str( obj ) ((char const *)(obj))
-#define object_copy( obj ) (obj)
-#define object_free( obj ) ((void)0)
-#define object_equal( lhs, rhs ) ((lhs) == (rhs))
-#define object_hash( obj ) (((struct hash_header *)((char *)(obj) - sizeof(struct hash_header)))->hash)
-
-#else
-
-char const * object_str ( OBJECT * );
-OBJECT * object_copy ( OBJECT * );
-void object_free ( OBJECT * );
-int object_equal( OBJECT *, OBJECT * );
-unsigned int object_hash ( OBJECT * );
-
-#endif
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/option.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/option.c b/ext/kenlm/jam-files/engine/option.c
deleted file mode 100644
index d25e5e8..0000000
--- a/ext/kenlm/jam-files/engine/option.c
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Copyright 1993, 1995 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-# include "jam.h"
-# include "option.h"
-
-/*
- * option.c - command line option processing
- *
- * {o >o
- * \<>) "Process command line options as defined in <option.h>.
- * Return the number of argv[] elements used up by options,
- * or -1 if an invalid option flag was given or an argument
- * was supplied for an option that does not require one."
- */
-
-int getoptions( int argc, char * * argv, char * opts, bjam_option * optv )
-{
- int i;
- int optc = N_OPTS;
-
- memset( (char *)optv, '\0', sizeof( *optv ) * N_OPTS );
-
- for ( i = 0; i < argc; ++i )
- {
- char *arg;
-
- if ( ( argv[ i ][ 0 ] != '-' ) ||
- ( ( argv[ i ][ 1 ] != '-' ) && !isalpha( argv[ i ][ 1 ] ) ) )
- continue;
-
- if ( !optc-- )
- {
- printf( "too many options (%d max)\n", N_OPTS );
- return -1;
- }
-
- for ( arg = &argv[ i ][ 1 ]; *arg; ++arg )
- {
- char * f;
-
- for ( f = opts; *f; ++f )
- if ( *f == *arg )
- break;
-
- if ( !*f )
- {
- printf( "Invalid option: -%c\n", *arg );
- return -1;
- }
-
- optv->flag = *f;
-
- if ( f[ 1 ] != ':' )
- {
- optv++->val = "true";
- }
- else if ( arg[ 1 ] )
- {
- optv++->val = &arg[1];
- break;
- }
- else if ( ++i < argc )
- {
- optv++->val = argv[ i ];
- break;
- }
- else
- {
- printf( "option: -%c needs argument\n", *f );
- return -1;
- }
- }
- }
-
- return i;
-}
-
-
-/*
- * Name: getoptval() - find an option given its character.
- */
-
-char * getoptval( bjam_option * optv, char opt, int subopt )
-{
- int i;
- for ( i = 0; i < N_OPTS; ++i, ++optv )
- if ( ( optv->flag == opt ) && !subopt-- )
- return optv->val;
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/option.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/option.h b/ext/kenlm/jam-files/engine/option.h
deleted file mode 100644
index 7c9c747..0000000
--- a/ext/kenlm/jam-files/engine/option.h
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright 1993, 1995 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/*
- * option.h - command line option processing
- *
- * {o >o
- * \ -) "Command line option."
- */
-
-typedef struct bjam_option
-{
- char flag; /* filled in by getoption() */
- char * val; /* set to random address if true */
-} bjam_option;
-
-#define N_OPTS 256
-
-int getoptions( int argc, char * * argv, char * opts, bjam_option * optv );
-char * getoptval( bjam_option * optv, char opt, int subopt );
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/output.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/output.c b/ext/kenlm/jam-files/engine/output.c
deleted file mode 100644
index eaaee43..0000000
--- a/ext/kenlm/jam-files/engine/output.c
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- Copyright 2007 Rene Rivera
- Distributed under the Boost Software License, Version 1.0.
- (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-*/
-
-#include "jam.h"
-#include "output.h"
-
-#include <stdio.h>
-
-
-#define bjam_out (stdout)
-#define bjam_err (stderr)
-
-static void out_( char const * data, FILE * const io )
-{
- while ( *data )
- {
- size_t const len = strcspn( data, "\r" );
- data += fwrite( data, 1, len, io );
- if ( *data == '\r' ) ++data;
- }
-}
-
-
-void out_action
-(
- char const * const action,
- char const * const target,
- char const * const command,
- char const * const out_data,
- char const * const err_data,
- int const exit_reason
-)
-{
- /* Print out the action + target line, if the action is quiet the action
- * should be null.
- */
- if ( action )
- fprintf( bjam_out, "%s %s\n", action, target );
-
- /* Print out the command executed if given -d+2. */
- if ( DEBUG_EXEC )
- {
- fputs( command, bjam_out );
- fputc( '\n', bjam_out );
- }
-
- /* Print out the command executed to the command stream. */
- if ( globs.cmdout )
- fputs( command, globs.cmdout );
-
- /* If the process expired, make user aware with an explicit message, but do
- * this only for non-quiet actions.
- */
- if ( exit_reason == EXIT_TIMEOUT && action )
- fprintf( bjam_out, "%ld second time limit exceeded\n", globs.timeout );
-
- /* Print out the command output, if requested, or if the program failed, but
- * only output for non-quiet actions.
- */
- if ( action || exit_reason != EXIT_OK )
- {
- if ( out_data &&
- ( ( globs.pipe_action & 1 /* STDOUT_FILENO */ ) ||
- ( globs.pipe_action == 0 ) ) )
- out_( out_data, bjam_out );
- if ( err_data && ( globs.pipe_action & 2 /* STDERR_FILENO */ ) )
- out_( err_data, bjam_err );
- }
-
- fflush( bjam_out );
- fflush( bjam_err );
- fflush( globs.cmdout );
-}
-
-
-OBJECT * outf_int( int const value )
-{
- char buffer[ 50 ];
- sprintf( buffer, "%i", value );
- return object_new( buffer );
-}
-
-
-OBJECT * outf_double( double const value )
-{
- char buffer[ 50 ];
- sprintf( buffer, "%f", value );
- return object_new( buffer );
-}
-
-
-OBJECT * outf_time( timestamp const * const time )
-{
- return object_new( timestamp_str( time ) );
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/output.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/output.h b/ext/kenlm/jam-files/engine/output.h
deleted file mode 100644
index 186e867..0000000
--- a/ext/kenlm/jam-files/engine/output.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- Copyright 2007 Rene Rivera
- Distributed under the Boost Software License, Version 1.0.
- (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
-*/
-
-#ifndef BJAM_OUTPUT_H
-#define BJAM_OUTPUT_H
-
-#include "object.h"
-#include "timestamp.h"
-
-#define EXIT_OK 0
-#define EXIT_FAIL 1
-#define EXIT_TIMEOUT 2
-
-void out_action(
- char const * const action,
- char const * const target,
- char const * const command,
- char const * const out_data,
- char const * const err_data,
- int const exit_reason
-);
-
-OBJECT * outf_int( int const value );
-OBJECT * outf_double( double const value );
-OBJECT * outf_time( timestamp const * const value );
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/parse.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/parse.c b/ext/kenlm/jam-files/engine/parse.c
deleted file mode 100644
index 02412e0..0000000
--- a/ext/kenlm/jam-files/engine/parse.c
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Copyright 1993, 2000 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#include "jam.h"
-#include "lists.h"
-#include "parse.h"
-#include "scan.h"
-#include "object.h"
-#include "modules.h"
-#include "frames.h"
-#include "function.h"
-
-/*
- * parse.c - make and destroy parse trees as driven by the parser
- *
- * 09/07/00 (seiwald) - ref count on PARSE to avoid freeing when used,
- * as per Matt Armstrong.
- * 09/11/00 (seiwald) - structure reworked to reflect that (*func)()
- * returns a LIST *.
- */
-
-static PARSE * yypsave;
-
-void parse_file( OBJECT * f, FRAME * frame )
-{
- /* Suspend scan of current file and push this new file in the stream. */
- yyfparse( f );
-
- /* Now parse each block of rules and execute it. Execute it outside of the
- * parser so that recursive calls to yyrun() work (no recursive yyparse's).
- */
-
- for ( ; ; )
- {
- PARSE * p;
- FUNCTION * func;
-
- /* Filled by yyparse() calling parse_save(). */
- yypsave = 0;
-
- /* If parse error or empty parse, outta here. */
- if ( yyparse() || !( p = yypsave ) )
- break;
-
- /* Run the parse tree. */
- func = function_compile( p );
- parse_free( p );
- list_free( function_run( func, frame, stack_global() ) );
- function_free( func );
- }
-}
-
-
-void parse_save( PARSE * p )
-{
- yypsave = p;
-}
-
-
-PARSE * parse_make(
- int type,
- PARSE * left,
- PARSE * right,
- PARSE * third,
- OBJECT * string,
- OBJECT * string1,
- int num )
-{
- PARSE * p = (PARSE *)BJAM_MALLOC( sizeof( PARSE ) );
-
- p->type = type;
- p->left = left;
- p->right = right;
- p->third = third;
- p->string = string;
- p->string1 = string1;
- p->num = num;
- p->refs = 1;
- p->rulename = 0;
-
- if ( left )
- {
- p->file = object_copy( left->file );
- p->line = left->line;
- }
- else
- {
- yyinput_last_read_token( &p->file, &p->line );
- p->file = object_copy( p->file );
- }
-
- return p;
-}
-
-
-void parse_refer( PARSE * p )
-{
- ++p->refs;
-}
-
-
-void parse_free( PARSE * p )
-{
- if ( --p->refs )
- return;
-
- if ( p->string )
- object_free( p->string );
- if ( p->string1 )
- object_free( p->string1 );
- if ( p->left )
- parse_free( p->left );
- if ( p->right )
- parse_free( p->right );
- if ( p->third )
- parse_free( p->third );
- if ( p->rulename )
- object_free( p->rulename );
- if ( p->file )
- object_free( p->file );
-
- BJAM_FREE( (char *)p );
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/parse.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/parse.h b/ext/kenlm/jam-files/engine/parse.h
deleted file mode 100644
index bb47af6..0000000
--- a/ext/kenlm/jam-files/engine/parse.h
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Copyright 1993, 2000 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * parse.h - make and destroy parse trees as driven by the parser.
- */
-
-#ifndef PARSE_DWA20011020_H
-#define PARSE_DWA20011020_H
-
-#include "frames.h"
-#include "lists.h"
-#include "modules.h"
-
-
-#define PARSE_APPEND 0
-#define PARSE_FOREACH 1
-#define PARSE_IF 2
-#define PARSE_EVAL 3
-#define PARSE_INCLUDE 4
-#define PARSE_LIST 5
-#define PARSE_LOCAL 6
-#define PARSE_MODULE 7
-#define PARSE_CLASS 8
-#define PARSE_NULL 9
-#define PARSE_ON 10
-#define PARSE_RULE 11
-#define PARSE_RULES 12
-#define PARSE_SET 13
-#define PARSE_SETCOMP 14
-#define PARSE_SETEXEC 15
-#define PARSE_SETTINGS 16
-#define PARSE_SWITCH 17
-#define PARSE_WHILE 18
-
-
-/*
- * Parse tree node.
- */
-
-typedef struct _PARSE PARSE;
-
-struct _PARSE {
- int type;
- PARSE * left;
- PARSE * right;
- PARSE * third;
- OBJECT * string;
- OBJECT * string1;
- int num;
- int refs;
- OBJECT * rulename;
- OBJECT * file;
- int line;
-};
-
-void parse_file( OBJECT *, FRAME * );
-void parse_save( PARSE * );
-
-PARSE * parse_make( int type, PARSE * left, PARSE * right, PARSE * third,
- OBJECT * string, OBJECT * string1, int num );
-
-void parse_refer( PARSE * );
-void parse_free( PARSE * );
-LIST * parse_evaluate( PARSE *, FRAME * );
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/patchlevel.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/patchlevel.h b/ext/kenlm/jam-files/engine/patchlevel.h
deleted file mode 100644
index 60b0d61..0000000
--- a/ext/kenlm/jam-files/engine/patchlevel.h
+++ /dev/null
@@ -1,17 +0,0 @@
-/*
- * Copyright 1993-2002 Christopher Seiwald and Perforce Software, Inc.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* Keep JAMVERSYM in sync with VERSION. */
-/* It can be accessed as $(JAMVERSION) in the Jamfile. */
-
-#define VERSION_MAJOR 2011
-#define VERSION_MINOR 12
-#define VERSION_PATCH 1
-#define VERSION_MAJOR_SYM "2011"
-#define VERSION_MINOR_SYM "12"
-#define VERSION_PATCH_SYM "01"
-#define VERSION "2011.12.1"
-#define JAMVERSYM "JAMVERSION=2011.12"
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/pathnt.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/pathnt.c b/ext/kenlm/jam-files/engine/pathnt.c
deleted file mode 100644
index 8abf8fe..0000000
--- a/ext/kenlm/jam-files/engine/pathnt.c
+++ /dev/null
@@ -1,301 +0,0 @@
-/*
- * Copyright 1993-2002 Christopher Seiwald and Perforce Software, Inc.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Copyright 2005 Rene Rivera.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * pathnt.c - NT specific path manipulation support
- */
-
-#include "pathsys.h"
-
-#include "hash.h"
-
-#define WIN32_LEAN_AND_MEAN
-#include <windows.h>
-
-#include <assert.h>
-#include <stdlib.h>
-
-
-/* The definition of this in winnt.h is not ANSI-C compatible. */
-#undef INVALID_FILE_ATTRIBUTES
-#define INVALID_FILE_ATTRIBUTES ((DWORD)-1)
-
-
-typedef struct path_key_entry
-{
- OBJECT * path;
- OBJECT * key;
- int exists;
-} path_key_entry;
-
-static struct hash * path_key_cache;
-
-
-/*
- * path_get_process_id_()
- */
-
-unsigned long path_get_process_id_( void )
-{
- return GetCurrentProcessId();
-}
-
-
-/*
- * path_get_temp_path_()
- */
-
-void path_get_temp_path_( string * buffer )
-{
- DWORD pathLength = GetTempPathA( 0, NULL );
- string_reserve( buffer, pathLength );
- pathLength = GetTempPathA( pathLength, buffer->value );
- buffer->value[ pathLength - 1 ] = '\0';
- buffer->size = pathLength - 1;
-}
-
-
-/*
- * canonicWindowsPath() - convert a given path into its canonic/long format
- *
- * Appends the canonic path to the end of the given 'string' object.
- *
- * FIXME: This function is still work-in-progress as it originally did not
- * necessarily return the canonic path format (could return slightly different
- * results for certain equivalent path strings) and could accept paths pointing
- * to non-existing file system entities as well.
- *
- * Caches results internally, automatically caching any parent paths it has to
- * convert to their canonic format in the process.
- *
- * Prerequisites:
- * - path given in normalized form, i.e. all of its folder separators have
- * already been converted into '\\'
- * - path_key_cache path/key mapping cache object already initialized
- */
-
-static int canonicWindowsPath( char const * const path, int const path_length,
- string * const out )
-{
- char const * last_element;
- unsigned long saved_size;
- char const * p;
- int missing_parent;
-
- /* This is only called via path_key(), which initializes the cache. */
- assert( path_key_cache );
-
- if ( !path_length )
- return 1;
-
- if ( path_length == 1 && path[ 0 ] == '\\' )
- {
- string_push_back( out, '\\' );
- return 1;
- }
-
- if ( path[ 1 ] == ':' &&
- ( path_length == 2 ||
- ( path_length == 3 && path[ 2 ] == '\\' ) ) )
- {
- string_push_back( out, toupper( path[ 0 ] ) );
- string_push_back( out, ':' );
- string_push_back( out, '\\' );
- return 1;
- }
-
- /* Find last '\\'. */
- for ( p = path + path_length - 1; p >= path && *p != '\\'; --p );
- last_element = p + 1;
-
- /* Special case '\' && 'D:\' - include trailing '\'. */
- if ( p == path ||
- p == path + 2 && path[ 1 ] == ':' )
- ++p;
-
- missing_parent = 0;
-
- if ( p >= path )
- {
- char const * const dir = path;
- int const dir_length = p - path;
- OBJECT * const dir_obj = object_new_range( dir, dir_length );
- int found;
- path_key_entry * const result = (path_key_entry *)hash_insert(
- path_key_cache, dir_obj, &found );
- if ( !found )
- {
- result->path = dir_obj;
- if ( canonicWindowsPath( dir, dir_length, out ) )
- result->exists = 1;
- else
- result->exists = 0;
- result->key = object_new( out->value );
- }
- else
- {
- object_free( dir_obj );
- string_append( out, object_str( result->key ) );
- }
- if ( !result->exists )
- missing_parent = 1;
- }
-
- if ( out->size && out->value[ out->size - 1 ] != '\\' )
- string_push_back( out, '\\' );
-
- saved_size = out->size;
- string_append_range( out, last_element, path + path_length );
-
- if ( !missing_parent )
- {
- char const * const n = last_element;
- int const n_length = path + path_length - n;
- if ( !( n_length == 1 && n[ 0 ] == '.' )
- && !( n_length == 2 && n[ 0 ] == '.' && n[ 1 ] == '.' ) )
- {
- WIN32_FIND_DATA fd;
- HANDLE const hf = FindFirstFileA( out->value, &fd );
- if ( hf != INVALID_HANDLE_VALUE )
- {
- string_truncate( out, saved_size );
- string_append( out, fd.cFileName );
- FindClose( hf );
- return 1;
- }
- }
- }
- return 0;
-}
-
-
-/*
- * normalize_path() - 'normalizes' the given path for the path-key mapping
- *
- * The resulting string has nothing to do with 'normalized paths' as used in
- * Boost Jam build scripts and the built-in NORMALIZE_PATH rule. It is intended
- * to be used solely as an intermediate step when mapping an arbitrary path to
- * its canonical representation.
- *
- * When choosing the intermediate string the important things are for it to be
- * inexpensive to calculate and any two paths having different canonical
- * representations also need to have different calculated intermediate string
- * representations. Any implemented additional rules serve only to simplify
- * constructing the canonical path representation from the calculated
- * intermediate string.
- *
- * Implemented returned path rules:
- * - use backslashes as path separators
- * - lowercase only (since all Windows file systems are case insensitive)
- * - trim trailing path separator except in case of a root path, i.e. 'X:\'
- */
-
-static void normalize_path( string * path )
-{
- char * s;
- for ( s = path->value; s < path->value + path->size; ++s )
- *s = *s == '/' ? '\\' : tolower( *s );
- /* Strip trailing "/". */
- if ( path->size && path->size != 3 && path->value[ path->size - 1 ] == '\\'
- )
- string_pop_back( path );
-}
-
-
-static path_key_entry * path_key( OBJECT * const path,
- int const known_to_be_canonic )
-{
- path_key_entry * result;
- int found;
-
- if ( !path_key_cache )
- path_key_cache = hashinit( sizeof( path_key_entry ), "path to key" );
-
- result = (path_key_entry *)hash_insert( path_key_cache, path, &found );
- if ( !found )
- {
- OBJECT * normalized;
- int normalized_size;
- path_key_entry * nresult;
- result->path = path;
- {
- string buf[ 1 ];
- string_copy( buf, object_str( path ) );
- normalize_path( buf );
- normalized = object_new( buf->value );
- normalized_size = buf->size;
- string_free( buf );
- }
- nresult = (path_key_entry *)hash_insert( path_key_cache, normalized,
- &found );
- if ( !found || nresult == result )
- {
- nresult->path = normalized;
- if ( known_to_be_canonic )
- nresult->key = object_copy( path );
- else
- {
- string canonic_path[ 1 ];
- string_new( canonic_path );
- if ( canonicWindowsPath( object_str( normalized ), normalized_size,
- canonic_path ) )
- nresult->exists = 1;
- else
- nresult->exists = 0;
- nresult->key = object_new( canonic_path->value );
- string_free( canonic_path );
- }
- }
- else
- object_free( normalized );
- if ( nresult != result )
- {
- result->path = object_copy( path );
- result->key = object_copy( nresult->key );
- result->exists = nresult->exists;
- }
- }
-
- return result;
-}
-
-
-void path_register_key( OBJECT * canonic_path )
-{
- path_key( canonic_path, 1 );
-}
-
-
-OBJECT * path_as_key( OBJECT * path )
-{
- return object_copy( path_key( path, 0 )->key );
-}
-
-
-static void free_path_key_entry( void * xentry, void * const data )
-{
- path_key_entry * const entry = (path_key_entry *)xentry;
- object_free( entry->path );
- object_free( entry->key );
-}
-
-
-void path_done( void )
-{
- if ( path_key_cache )
- {
- hashenumerate( path_key_cache, &free_path_key_entry, 0 );
- hashdone( path_key_cache );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/pathsys.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/pathsys.c b/ext/kenlm/jam-files/engine/pathsys.c
deleted file mode 100644
index ae4e6e0..0000000
--- a/ext/kenlm/jam-files/engine/pathsys.c
+++ /dev/null
@@ -1,285 +0,0 @@
-/*
- * Copyright 1993-2002 Christopher Seiwald and Perforce Software, Inc.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Copyright 2005 Rene Rivera.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * pathsys.c - platform independent path manipulation support
- *
- * External routines:
- * path_build() - build a filename given dir/base/suffix/member
- * path_parent() - make a PATHNAME point to its parent dir
- * path_parse() - split a file name into dir/base/suffix/member
- * path_tmpdir() - returns the system dependent temporary folder path
- * path_tmpfile() - returns a new temporary path
- * path_tmpnam() - returns a new temporary name
- *
- * File_parse() and path_build() just manipulate a string and a structure;
- * they do not make system calls.
- */
-
-#include "jam.h"
-#include "pathsys.h"
-
-#include "filesys.h"
-
-#include <stdlib.h>
-#include <time.h>
-
-
-/* Internal OS specific implementation details - have names ending with an
- * underscore and are expected to be implemented in an OS specific pathXXX.c
- * module.
- */
-unsigned long path_get_process_id_( void );
-void path_get_temp_path_( string * buffer );
-
-
-/*
- * path_parse() - split a file name into dir/base/suffix/member
- */
-
-void path_parse( char const * file, PATHNAME * f )
-{
- char const * p;
- char const * q;
- char const * end;
-
- memset( (char *)f, 0, sizeof( *f ) );
-
- /* Look for '<grist>'. */
-
- if ( ( file[ 0 ] == '<' ) && ( p = strchr( file, '>' ) ) )
- {
- f->f_grist.ptr = file;
- f->f_grist.len = p - file;
- file = p + 1;
- }
-
- /* Look for 'dir/'. */
-
- p = strrchr( file, '/' );
-
-#if PATH_DELIM == '\\'
- /* On NT, look for dir\ as well */
- {
- char * const p1 = strrchr( p ? p + 1 : file, '\\' );
- if ( p1 ) p = p1;
- }
-#endif
-
- if ( p )
- {
- f->f_dir.ptr = file;
- f->f_dir.len = p - file;
-
- /* Special case for / - dirname is /, not "" */
- if ( !f->f_dir.len )
- ++f->f_dir.len;
-
-#if PATH_DELIM == '\\'
- /* Special case for D:/ - dirname is D:/, not "D:" */
- if ( f->f_dir.len == 2 && file[ 1 ] == ':' )
- ++f->f_dir.len;
-#endif
-
- file = p + 1;
- }
-
- end = file + strlen( file );
-
- /* Look for '(member)'. */
- if ( ( p = strchr( file, '(' ) ) && ( end[ -1 ] == ')' ) )
- {
- f->f_member.ptr = p + 1;
- f->f_member.len = end - p - 2;
- end = p;
- }
-
- /* Look for '.suffix'. This would be memrchr(). */
- p = 0;
- for ( q = file; ( q = (char *)memchr( q, '.', end - q ) ); ++q )
- p = q;
- if ( p )
- {
- f->f_suffix.ptr = p;
- f->f_suffix.len = end - p;
- end = p;
- }
-
- /* Leaves base. */
- f->f_base.ptr = file;
- f->f_base.len = end - file;
-}
-
-
-/*
- * is_path_delim() - true iff c is a path delimiter
- */
-
-static int is_path_delim( char const c )
-{
- return c == PATH_DELIM
-#if PATH_DELIM == '\\'
- || c == '/'
-#endif
- ;
-}
-
-
-/*
- * as_path_delim() - convert c to a path delimiter if it is not one already
- */
-
-static char as_path_delim( char const c )
-{
- return is_path_delim( c ) ? c : PATH_DELIM;
-}
-
-
-/*
- * path_build() - build a filename given dir/base/suffix/member
- *
- * To avoid changing slash direction on NT when reconstituting paths, instead of
- * unconditionally appending PATH_DELIM we check the past-the-end character of
- * the previous path element. If it is a path delimiter, we append that, and
- * only append PATH_DELIM as a last resort. This heuristic is based on the fact
- * that PATHNAME objects are usually the result of calling path_parse, which
- * leaves the original slashes in the past-the-end position. Correctness depends
- * on the assumption that all strings are zero terminated, so a past-the-end
- * character will always be available.
- *
- * As an attendant patch, we had to ensure that backslashes are used explicitly
- * in 'timestamp.c'.
- */
-
-void path_build( PATHNAME * f, string * file )
-{
- file_build1( f, file );
-
- /* Do not prepend root if it is '.' or the directory is rooted. */
- if ( f->f_root.len
- && !( f->f_root.len == 1 && f->f_root.ptr[ 0 ] == '.' )
- && !( f->f_dir.len && f->f_dir.ptr[ 0 ] == '/' )
-#if PATH_DELIM == '\\'
- && !( f->f_dir.len && f->f_dir.ptr[ 0 ] == '\\' )
- && !( f->f_dir.len && f->f_dir.ptr[ 1 ] == ':' )
-#endif
- )
- {
- string_append_range( file, f->f_root.ptr, f->f_root.ptr + f->f_root.len
- );
- /* If 'root' already ends with a path delimeter, do not add another one.
- */
- if ( !is_path_delim( f->f_root.ptr[ f->f_root.len - 1 ] ) )
- string_push_back( file, as_path_delim( f->f_root.ptr[ f->f_root.len
- ] ) );
- }
-
- if ( f->f_dir.len )
- string_append_range( file, f->f_dir.ptr, f->f_dir.ptr + f->f_dir.len );
-
- /* Put path separator between dir and file. */
- /* Special case for root dir: do not add another path separator. */
- if ( f->f_dir.len && ( f->f_base.len || f->f_suffix.len )
-#if PATH_DELIM == '\\'
- && !( f->f_dir.len == 3 && f->f_dir.ptr[ 1 ] == ':' )
-#endif
- && !( f->f_dir.len == 1 && is_path_delim( f->f_dir.ptr[ 0 ] ) ) )
- string_push_back( file, as_path_delim( f->f_dir.ptr[ f->f_dir.len ] ) );
-
- if ( f->f_base.len )
- string_append_range( file, f->f_base.ptr, f->f_base.ptr + f->f_base.len
- );
-
- if ( f->f_suffix.len )
- string_append_range( file, f->f_suffix.ptr, f->f_suffix.ptr +
- f->f_suffix.len );
-
- if ( f->f_member.len )
- {
- string_push_back( file, '(' );
- string_append_range( file, f->f_member.ptr, f->f_member.ptr +
- f->f_member.len );
- string_push_back( file, ')' );
- }
-}
-
-
-/*
- * path_parent() - make a PATHNAME point to its parent dir
- */
-
-void path_parent( PATHNAME * f )
-{
- f->f_base.ptr = f->f_suffix.ptr = f->f_member.ptr = "";
- f->f_base.len = f->f_suffix.len = f->f_member.len = 0;
-}
-
-
-/*
- * path_tmpdir() - returns the system dependent temporary folder path
- *
- * Returned value is stored inside a static buffer and should not be modified.
- * Returned value does *not* include a trailing path separator.
- */
-
-string const * path_tmpdir()
-{
- static string buffer[ 1 ];
- static int have_result;
- if ( !have_result )
- {
- string_new( buffer );
- path_get_temp_path_( buffer );
- have_result = 1;
- }
- return buffer;
-}
-
-
-/*
- * path_tmpnam() - returns a new temporary name
- */
-
-OBJECT * path_tmpnam( void )
-{
- char name_buffer[ 64 ];
- unsigned long const pid = path_get_process_id_();
- static unsigned long t;
- if ( !t ) t = time( 0 ) & 0xffff;
- t += 1;
- sprintf( name_buffer, "jam%lx%lx.000", pid, t );
- return object_new( name_buffer );
-}
-
-
-/*
- * path_tmpfile() - returns a new temporary path
- */
-
-OBJECT * path_tmpfile( void )
-{
- OBJECT * result;
- OBJECT * tmpnam;
-
- string file_path[ 1 ];
- string_copy( file_path, path_tmpdir()->value );
- string_push_back( file_path, PATH_DELIM );
- tmpnam = path_tmpnam();
- string_append( file_path, object_str( tmpnam ) );
- object_free( tmpnam );
- result = object_new( file_path->value );
- string_free( file_path );
-
- return result;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/pathsys.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/pathsys.h b/ext/kenlm/jam-files/engine/pathsys.h
deleted file mode 100644
index 9b7a4ca..0000000
--- a/ext/kenlm/jam-files/engine/pathsys.h
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Copyright 1993-2002 Christopher Seiwald and Perforce Software, Inc.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/*
- * pathsys.h - PATHNAME struct
- */
-
-/*
- * PATHNAME - a name of a file, broken into <grist>dir/base/suffix(member)
- *
- * <grist> - salt to distinguish between targets that would otherwise have the
- * same name - it never appears in the bound name of a target.
- *
- * (member) - archive member name: the syntax is arbitrary, but must agree in
- * path_parse(), path_build() and the Jambase.
- */
-
-#ifndef PATHSYS_VP_20020211_H
-#define PATHSYS_VP_20020211_H
-
-#include "object.h"
-#include "strings.h"
-
-
-typedef struct _pathpart
-{
- char const * ptr;
- int len;
-} PATHPART;
-
-typedef struct _pathname
-{
- PATHPART part[ 6 ];
-
-#define f_grist part[ 0 ]
-#define f_root part[ 1 ]
-#define f_dir part[ 2 ]
-#define f_base part[ 3 ]
-#define f_suffix part[ 4 ]
-#define f_member part[ 5 ]
-} PATHNAME;
-
-
-void path_build( PATHNAME *, string * file );
-void path_parse( char const * file, PATHNAME * );
-void path_parent( PATHNAME * );
-
-/* Given a path, returns an object containing an equivalent path in canonical
- * format that can be used as a unique key for that path. Equivalent paths such
- * as a/b, A\B, and a\B on NT all yield the same key.
- */
-OBJECT * path_as_key( OBJECT * path );
-
-/* Called as an optimization when we know we have a path that is already in its
- * canonical/long/key form. Avoids the need for some subsequent path_as_key()
- * call to do a potentially expensive path conversion requiring access to the
- * actual underlying file system.
- */
-void path_register_key( OBJECT * canonic_path );
-
-/* Returns a static pointer to the system dependent path to the temporary
- * directory. NOTE: Does *not* include a trailing path separator.
- */
-string const * path_tmpdir( void );
-
-/* Returns a new temporary name. */
-OBJECT * path_tmpnam( void );
-
-/* Returns a new temporary path. */
-OBJECT * path_tmpfile( void );
-
-/* Give the first argument to 'main', return a full path to our executable.
- * Returns null in the unlikely case it cannot be determined. Caller is
- * responsible for freeing the string.
- *
- * Implemented in jam.c
- */
-char * executable_path( char const * argv0 );
-
-void path_done( void );
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/pathunix.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/pathunix.c b/ext/kenlm/jam-files/engine/pathunix.c
deleted file mode 100644
index 8ca0d18..0000000
--- a/ext/kenlm/jam-files/engine/pathunix.c
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Copyright 1993-2002 Christopher Seiwald and Perforce Software, Inc.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Copyright 2005 Rene Rivera.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * pathunix.c - UNIX specific path manipulation support
- */
-
-#include "pathsys.h"
-
-#include <stdlib.h>
-#include <unistd.h> /* needed for getpid() */
-
-
-/*
- * path_get_process_id_()
- */
-
-unsigned long path_get_process_id_( void )
-{
- return getpid();
-}
-
-
-/*
- * path_get_temp_path_()
- */
-
-void path_get_temp_path_( string * buffer )
-{
- char const * t = getenv( "TMPDIR" );
- string_append( buffer, t ? t : "/tmp" );
-}
-
-
-/*
- * path_register_key()
- */
-
-void path_register_key( OBJECT * path )
-{
-}
-
-
-/*
- * path_as_key()
- */
-
-OBJECT * path_as_key( OBJECT * path )
-{
- return object_copy( path );
-}
-
-
-/*
- * path_done()
- */
-
-void path_done( void )
-{
-}