You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/04/19 21:34:07 UTC
[19/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/make1.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/make1.c b/ext/kenlm/jam-files/engine/make1.c
deleted file mode 100644
index 71eee12..0000000
--- a/ext/kenlm/jam-files/engine/make1.c
+++ /dev/null
@@ -1,1283 +0,0 @@
-/*
- * Copyright 1993-2002 Christopher Seiwald and Perforce Software, Inc.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * make1.c - execute commands to bring targets up to date
- *
- * This module contains make1(), the entry point called by make() to recursively
- * descend the dependency graph executing update actions as marked by make0().
- *
- * External routines:
- * make1() - execute commands to update a TARGET and all of its dependencies
- *
- * Internal routines, the recursive/asynchronous command executors:
- * make1a() - recursively schedules dependency builds and then goes to
- * MAKE1B
- * make1b() - if nothing is blocking this target's build, proceed to
- * MAKE1C
- * make1c() - launch target's next command, or go to parents' MAKE1B
- * if none
- * make1c_closure() - handle command execution completion and go to MAKE1C
- *
- * Internal support routines:
- * make1cmds() - turn ACTIONS into CMDs, grouping, splitting, etc.
- * make1list() - turn a list of targets into a LIST, for $(<) and $(>)
- * make1settings() - for vars with bound values, build up replacement lists
- * make1bind() - bind targets that weren't bound in dependency analysis
- */
-
-#include "jam.h"
-#include "make.h"
-
-#include "command.h"
-#include "compile.h"
-#include "execcmd.h"
-#include "headers.h"
-#include "lists.h"
-#include "object.h"
-#include "output.h"
-#include "parse.h"
-#include "rules.h"
-#include "search.h"
-#include "variable.h"
-
-#include <assert.h>
-#include <stdlib.h>
-
-#if !defined( NT ) || defined( __GNUC__ )
- #include <unistd.h> /* for unlink */
-#endif
-
-static CMD * make1cmds ( TARGET * );
-static LIST * make1list ( LIST *, TARGETS *, int flags );
-static SETTINGS * make1settings ( struct module_t *, LIST * vars );
-static void make1bind ( TARGET * );
-static TARGET * make1findcycle ( TARGET * );
-static void make1breakcycle( TARGET *, TARGET * cycle_root );
-
-/* Ugly static - it is too hard to carry it through the callbacks. */
-
-static struct
-{
- int failed;
- int skipped;
- int total;
- int made;
-} counts[ 1 ];
-
-/* Target state. */
-#define T_STATE_MAKE1A 0 /* make1a() should be called */
-#define T_STATE_MAKE1B 1 /* make1b() should be called */
-#define T_STATE_MAKE1C 2 /* make1c() should be called */
-
-typedef struct _state state;
-struct _state
-{
- state * prev; /* previous state on stack */
- TARGET * t; /* current target */
- TARGET * parent; /* parent argument necessary for MAKE1A */
- int curstate; /* current state */
-};
-
-static void make1a( state * const );
-static void make1b( state * const );
-static void make1c( state const * const );
-
-static void make1c_closure( void * const closure, int status,
- timing_info const * const, char const * const cmd_stdout,
- char const * const cmd_stderr, int const cmd_exit_reason );
-
-typedef struct _stack
-{
- state * stack;
-} stack;
-
-static stack state_stack = { NULL };
-
-static state * state_freelist = NULL;
-
-/* Currently running command counter. */
-static int cmdsrunning;
-
-
-static state * alloc_state()
-{
- if ( state_freelist )
- {
- state * const pState = state_freelist;
- state_freelist = pState->prev;
- memset( pState, 0, sizeof( state ) );
- return pState;
- }
- return (state *)BJAM_MALLOC( sizeof( state ) );
-}
-
-
-static void free_state( state * const pState )
-{
- pState->prev = state_freelist;
- state_freelist = pState;
-}
-
-
-static void clear_state_freelist()
-{
- while ( state_freelist )
- {
- state * const pState = state_freelist;
- state_freelist = state_freelist->prev;
- BJAM_FREE( pState );
- }
-}
-
-
-static state * current_state( stack * const pStack )
-{
- return pStack->stack;
-}
-
-
-static void pop_state( stack * const pStack )
-{
- if ( pStack->stack )
- {
- state * const pState = pStack->stack->prev;
- free_state( pStack->stack );
- pStack->stack = pState;
- }
-}
-
-
-static state * push_state( stack * const pStack, TARGET * const t,
- TARGET * const parent, int const curstate )
-{
- state * const pState = alloc_state();
- pState->t = t;
- pState->parent = parent;
- pState->prev = pStack->stack;
- pState->curstate = curstate;
- return pStack->stack = pState;
-}
-
-
-/*
- * Pushes a stack onto another stack, effectively reversing the order.
- */
-
-static void push_stack_on_stack( stack * const pDest, stack * const pSrc )
-{
- while ( pSrc->stack )
- {
- state * const pState = pSrc->stack;
- pSrc->stack = pState->prev;
- pState->prev = pDest->stack;
- pDest->stack = pState;
- }
-}
-
-
-/*
- * make1() - execute commands to update a list of targets and all of their dependencies
- */
-
-static int intr = 0;
-static int quit = 0;
-
-int make1( LIST * targets )
-{
- state * pState;
- int status = 0;
-
- memset( (char *)counts, 0, sizeof( *counts ) );
-
- {
- LISTITER iter, end;
- stack temp_stack = { NULL };
- for ( iter = list_begin( targets ), end = list_end( targets );
- iter != end; iter = list_next( iter ) )
- push_state( &temp_stack, bindtarget( list_item( iter ) ), NULL, T_STATE_MAKE1A );
- push_stack_on_stack( &state_stack, &temp_stack );
- }
-
- /* Clear any state left over from the past */
- quit = 0;
-
- /* Recursively make the target and its dependencies. */
-
- while ( 1 )
- {
- while ( ( pState = current_state( &state_stack ) ) )
- {
- if ( quit )
- pop_state( &state_stack );
-
- switch ( pState->curstate )
- {
- case T_STATE_MAKE1A: make1a( pState ); break;
- case T_STATE_MAKE1B: make1b( pState ); break;
- case T_STATE_MAKE1C: make1c( pState ); break;
- default:
- assert( !"make1(): Invalid state detected." );
- }
- }
- if ( !cmdsrunning )
- break;
- /* Wait for outstanding commands to finish running. */
- exec_wait();
- }
-
- clear_state_freelist();
-
- /* Talk about it. */
- if ( counts->failed )
- printf( "...failed updating %d target%s...\n", counts->failed,
- counts->failed > 1 ? "s" : "" );
- if ( DEBUG_MAKE && counts->skipped )
- printf( "...skipped %d target%s...\n", counts->skipped,
- counts->skipped > 1 ? "s" : "" );
- if ( DEBUG_MAKE && counts->made )
- printf( "...updated %d target%s...\n", counts->made,
- counts->made > 1 ? "s" : "" );
-
- /* If we were interrupted, exit now that all child processes
- have finished. */
- if ( intr )
- exit( 1 );
-
- {
- LISTITER iter, end;
- for ( iter = list_begin( targets ), end = list_end( targets );
- iter != end; iter = list_next( iter ) )
- {
- /* Check that the target was updated and that the
- update succeeded. */
- TARGET * t = bindtarget( list_item( iter ) );
- if (t->progress == T_MAKE_DONE)
- {
- if (t->status != EXEC_CMD_OK)
- status = 1;
- }
- else if ( ! ( t->progress == T_MAKE_NOEXEC_DONE && globs.noexec ) )
- {
- status = 1;
- }
- }
- }
- return status;
-}
-
-
-/*
- * make1a() - recursively schedules dependency builds and then goes to MAKE1B
- *
- * Called to start processing a specified target. Does nothing if the target is
- * already being processed or otherwise starts processing all of its
- * dependencies.
- */
-
-static void make1a( state * const pState )
-{
- TARGET * t = pState->t;
- TARGET * const scc_root = target_scc( t );
-
- if ( !pState->parent || target_scc( pState->parent ) != scc_root )
- pState->t = t = scc_root;
-
- /* If the parent is the first to try to build this target or this target is
- * in the MAKE1C quagmire, arrange for the parent to be notified when this
- * target has been built.
- */
- if ( pState->parent && t->progress <= T_MAKE_RUNNING )
- {
- TARGET * const parent_scc = target_scc( pState->parent );
- if ( t != parent_scc )
- {
- t->parents = targetentry( t->parents, parent_scc );
- ++parent_scc->asynccnt;
- }
- }
-
- /* If the target has been previously updated with -n in effect, and we are
- * now ignoring -n, update it for real. E.g. if the UPDATE_NOW rule was
- * called for it twice - first with the -n option and then without.
- */
- if ( !globs.noexec && t->progress == T_MAKE_NOEXEC_DONE )
- t->progress = T_MAKE_INIT;
-
- /* If this target is already being processed then do nothing. There is no
- * need to start processing the same target all over again.
- */
- if ( t->progress != T_MAKE_INIT )
- {
- pop_state( &state_stack );
- return;
- }
-
- /* Guard against circular dependencies. */
- t->progress = T_MAKE_ONSTACK;
-
- /* 'asynccnt' counts the dependencies preventing this target from proceeding
- * to MAKE1C for actual building. We start off with a count of 1 to prevent
- * anything from happening until we can notify all dependencies that they
- * are needed. This 1 is then accounted for when we enter MAKE1B ourselves,
- * below. Without this if a dependency gets built before we finish
- * processing all of our other dependencies our build might be triggerred
- * prematurely.
- */
- t->asynccnt = 1;
-
- /* Push dependency build requests (to be executed in the natural order). */
- {
- stack temp_stack = { NULL };
- TARGETS * c;
- for ( c = t->depends; c && !quit; c = c->next )
- push_state( &temp_stack, c->target, t, T_STATE_MAKE1A );
- push_stack_on_stack( &state_stack, &temp_stack );
- }
-
- t->progress = T_MAKE_ACTIVE;
-
- /* Once all of our dependencies have started getting processed we can move
- * onto MAKE1B.
- */
- /* Implementation note:
- * In theory this would be done by popping this state before pushing
- * dependency target build requests but as a slight optimization we simply
- * modify our current state and leave it on the stack instead.
- */
- pState->curstate = T_STATE_MAKE1B;
-}
-
-
-/*
- * make1b() - if nothing is blocking this target's build, proceed to MAKE1C
- *
- * Called after something stops blocking this target's build, e.g. that all of
- * its dependencies have started being processed, one of its dependencies has
- * been built or a semaphore this target has been waiting for is free again.
- */
-
-static void make1b( state * const pState )
-{
- TARGET * const t = pState->t;
- TARGET * failed = 0;
- char const * failed_name = "dependencies";
-
- /* If any dependencies are still outstanding, wait until they signal their
- * completion by pushing this same state for their parent targets.
- */
- if ( --t->asynccnt )
- {
- pop_state( &state_stack );
- return;
- }
-
- /* Try to aquire a semaphore. If it is locked, wait until the target that
- * locked it is built and signals completition.
- */
-#ifdef OPT_SEMAPHORE
- if ( t->semaphore && t->semaphore->asynccnt )
- {
- /* Append 't' to the list of targets waiting on semaphore. */
- t->semaphore->parents = targetentry( t->semaphore->parents, t );
- t->asynccnt++;
-
- if ( DEBUG_EXECCMD )
- printf( "SEM: %s is busy, delaying launch of %s\n",
- object_str( t->semaphore->name ), object_str( t->name ) );
- pop_state( &state_stack );
- return;
- }
-#endif
-
- /* Now ready to build target 't', if dependencies built OK. */
-
- /* Collect status from dependencies. If -n was passed then act as though all
- * dependencies built correctly (the only way they can fail is if UPDATE_NOW
- * was called). If the dependencies can not be found or we got an interrupt,
- * we can not get here.
- */
- if ( !globs.noexec )
- {
- TARGETS * c;
- for ( c = t->depends; c; c = c->next )
- if ( c->target->status > t->status && !( c->target->flags &
- T_FLAG_NOCARE ) )
- {
- failed = c->target;
- t->status = c->target->status;
- }
- }
-
- /* If an internal header node failed to build, we want to output the target
- * that it failed on.
- */
- if ( failed )
- failed_name = failed->flags & T_FLAG_INTERNAL
- ? failed->failed
- : object_str( failed->name );
- t->failed = failed_name;
-
- /* If actions for building any of the dependencies have failed, bail.
- * Otherwise, execute all actions to make the current target.
- */
- if ( ( t->status == EXEC_CMD_FAIL ) && t->actions )
- {
- ++counts->skipped;
- if ( ( t->flags & ( T_FLAG_RMOLD | T_FLAG_NOTFILE ) ) == T_FLAG_RMOLD )
- {
- if ( !unlink( object_str( t->boundname ) ) )
- printf( "...removing outdated %s\n", object_str( t->boundname )
- );
- }
- else
- printf( "...skipped %s for lack of %s...\n", object_str( t->name ),
- failed_name );
- }
-
- if ( t->status == EXEC_CMD_OK )
- switch ( t->fate )
- {
- case T_FATE_STABLE:
- case T_FATE_NEWER:
- break;
-
- case T_FATE_CANTFIND:
- case T_FATE_CANTMAKE:
- t->status = EXEC_CMD_FAIL;
- break;
-
- case T_FATE_ISTMP:
- if ( DEBUG_MAKE )
- printf( "...using %s...\n", object_str( t->name ) );
- break;
-
- case T_FATE_TOUCHED:
- case T_FATE_MISSING:
- case T_FATE_NEEDTMP:
- case T_FATE_OUTDATED:
- case T_FATE_UPDATE:
- case T_FATE_REBUILD:
- /* Prepare commands for executing actions scheduled for this target.
- * Commands have their embedded variables automatically expanded,
- * including making use of any "on target" variables.
- */
- if ( t->actions )
- {
- ++counts->total;
- if ( DEBUG_MAKE && !( counts->total % 100 ) )
- printf( "...on %dth target...\n", counts->total );
-
- t->cmds = (char *)make1cmds( t );
- /* Update the target's "progress" so MAKE1C processing counts it
- * among its successes/failures.
- */
- t->progress = T_MAKE_RUNNING;
- }
- break;
-
- /* All valid fates should have been accounted for by now. */
- default:
- printf( "ERROR: %s has bad fate %d", object_str( t->name ),
- t->fate );
- abort();
- }
-
-#ifdef OPT_SEMAPHORE
- /* If there is a semaphore, indicate that it is in use. */
- if ( t->semaphore )
- {
- ++t->semaphore->asynccnt;
- if ( DEBUG_EXECCMD )
- printf( "SEM: %s now used by %s\n", object_str( t->semaphore->name
- ), object_str( t->name ) );
- }
-#endif
-
- /* Proceed to MAKE1C to begin executing the chain of commands prepared for
- * building the target. If we are not going to build the target (e.g. due to
- * dependency failures or no commands needing to be run) the chain will be
- * empty and MAKE1C processing will directly signal the target's completion.
- */
- /* Implementation note:
- * Morfing the current state on the stack instead of popping it and
- * pushing a new one is a slight optimization with no side-effects since we
- * pushed no other states while processing this one.
- */
- pState->curstate = T_STATE_MAKE1C;
-}
-
-
-/*
- * make1c() - launch target's next command, or go to parents' MAKE1B if none
- *
- * If there are (more) commands to run to build this target (and we have not hit
- * an error running earlier comands) we launch the command using exec_cmd().
- * Command execution signals its completion in exec_wait() by calling our
- * make1c_closure() callback.
- *
- * If there are no more commands to run, we collect the status from all the
- * actions and report our completion to all the parents.
- */
-
-static void make1c( state const * const pState )
-{
- TARGET * const t = pState->t;
- CMD * const cmd = (CMD *)t->cmds;
-
- if ( cmd && t->status == EXEC_CMD_OK )
- {
- /* Pop state first in case something below (e.g. exec_cmd(), exec_wait()
- * or make1c_closure()) pushes a new state. Note that we must not access
- * the popped state data after this as the same stack node might have
- * been reused internally for some newly pushed state.
- */
- pop_state( &state_stack );
-
- /* Increment the jobs running counter. */
- ++cmdsrunning;
-
- /* Execute the actual build command or fake it if no-op. */
- if ( globs.noexec || cmd->noop )
- {
- timing_info time_info = { 0 };
- timestamp_current( &time_info.start );
- timestamp_copy( &time_info.end, &time_info.start );
- make1c_closure( t, EXEC_CMD_OK, &time_info, "", "", EXIT_OK );
- }
- else
- {
- exec_cmd( cmd->buf, make1c_closure, t, cmd->shell );
-
- /* Wait until under the concurrent command count limit. */
- /* FIXME: This wait could be skipped here and moved to just before
- * trying to execute a command that would cross the command count
- * limit. Note though that this might affect the order in which
- * unrelated targets get built and would thus require that all
- * affected Boost Build tests be updated.
- */
- assert( 0 < globs.jobs );
- assert( globs.jobs <= MAXJOBS );
- while ( cmdsrunning >= globs.jobs )
- exec_wait();
- }
- }
- else
- {
- ACTIONS * actions;
-
- /* Collect status from actions, and distribute it as well. */
- for ( actions = t->actions; actions; actions = actions->next )
- if ( actions->action->status > t->status )
- t->status = actions->action->status;
- for ( actions = t->actions; actions; actions = actions->next )
- if ( t->status > actions->action->status )
- actions->action->status = t->status;
-
- /* Tally success/failure for those we tried to update. */
- if ( t->progress == T_MAKE_RUNNING )
- switch ( t->status )
- {
- case EXEC_CMD_OK: ++counts->made; break;
- case EXEC_CMD_FAIL: ++counts->failed; break;
- }
-
- /* Tell parents their dependency has been built. */
- {
- TARGETS * c;
- stack temp_stack = { NULL };
- TARGET * additional_includes = NULL;
-
- t->progress = globs.noexec ? T_MAKE_NOEXEC_DONE : T_MAKE_DONE;
-
- /* Target has been updated so rescan it for dependencies. */
- if ( t->fate >= T_FATE_MISSING && t->status == EXEC_CMD_OK &&
- !( t->flags & T_FLAG_INTERNAL ) )
- {
- TARGET * saved_includes;
- SETTINGS * s;
-
- t->rescanned = 1;
-
- /* Clean current includes. */
- saved_includes = t->includes;
- t->includes = 0;
-
- s = copysettings( t->settings );
- pushsettings( root_module(), s );
- headers( t );
- popsettings( root_module(), s );
- freesettings( s );
-
- if ( t->includes )
- {
- /* Tricky. The parents have already been processed, but they
- * have not seen the internal node, because it was just
- * created. We need to:
- * - push MAKE1A states that would have been pushed by the
- * parents here
- * - make sure all unprocessed parents will pick up the
- * new includes
- * - make sure processing the additional MAKE1A states is
- * done before processing the MAKE1B state for our
- * current target (which would mean this target has
- * already been built), otherwise the parent would be
- * considered built before the additional MAKE1A state
- * processing even got a chance to start.
- */
- make0( t->includes, t->parents->target, 0, 0, 0, t->includes
- );
- /* Link the old includes on to make sure that it gets
- * cleaned up correctly.
- */
- t->includes->includes = saved_includes;
- for ( c = t->dependants; c; c = c->next )
- c->target->depends = targetentry( c->target->depends,
- t->includes );
- /* Will be processed below. */
- additional_includes = t->includes;
- }
- else
- {
- t->includes = saved_includes;
- }
- }
-
- if ( additional_includes )
- for ( c = t->parents; c; c = c->next )
- push_state( &temp_stack, additional_includes, c->target,
- T_STATE_MAKE1A );
-
- if ( t->scc_root )
- {
- TARGET * const scc_root = target_scc( t );
- assert( scc_root->progress < T_MAKE_DONE );
- for ( c = t->parents; c; c = c->next )
- {
- if ( target_scc( c->target ) == scc_root )
- push_state( &temp_stack, c->target, NULL, T_STATE_MAKE1B
- );
- else
- scc_root->parents = targetentry( scc_root->parents,
- c->target );
- }
- }
- else
- {
- for ( c = t->parents; c; c = c->next )
- push_state( &temp_stack, c->target, NULL, T_STATE_MAKE1B );
- }
-
-#ifdef OPT_SEMAPHORE
- /* If there is a semaphore, it is now free. */
- if ( t->semaphore )
- {
- assert( t->semaphore->asynccnt == 1 );
- --t->semaphore->asynccnt;
-
- if ( DEBUG_EXECCMD )
- printf( "SEM: %s is now free\n", object_str(
- t->semaphore->name ) );
-
- /* If anything is waiting, notify the next target. There is no
- * point in notifying all waiting targets, since they will be
- * notified again.
- */
- if ( t->semaphore->parents )
- {
- TARGETS * first = t->semaphore->parents;
- t->semaphore->parents = first->next;
- if ( first->next )
- first->next->tail = first->tail;
-
- if ( DEBUG_EXECCMD )
- printf( "SEM: placing %s on stack\n", object_str(
- first->target->name ) );
- push_state( &temp_stack, first->target, NULL, T_STATE_MAKE1B
- );
- BJAM_FREE( first );
- }
- }
-#endif
-
- /* Must pop state before pushing any more. */
- pop_state( &state_stack );
-
- /* Using stacks reverses the order of execution. Reverse it back. */
- push_stack_on_stack( &state_stack, &temp_stack );
- }
- }
-}
-
-
-/*
- * call_timing_rule() - Look up the __TIMING_RULE__ variable on the given
- * target, and if non-empty, invoke the rule it names, passing the given
- * timing_info.
- */
-
-static void call_timing_rule( TARGET * target, timing_info const * const time )
-{
- LIST * timing_rule;
-
- pushsettings( root_module(), target->settings );
- timing_rule = var_get( root_module(), constant_TIMING_RULE );
- popsettings( root_module(), target->settings );
-
- if ( !list_empty( timing_rule ) )
- {
- /* rule timing-rule ( args * : target : start end user system ) */
-
- /* Prepare the argument list. */
- FRAME frame[ 1 ];
- OBJECT * rulename = list_front( timing_rule );
- frame_init( frame );
-
- /* args * :: $(__TIMING_RULE__[2-]) */
- lol_add( frame->args, list_copy_range( timing_rule, list_next(
- list_begin( timing_rule ) ), list_end( timing_rule ) ) );
-
- /* target :: the name of the target */
- lol_add( frame->args, list_new( object_copy( target->name ) ) );
-
- /* start end user system :: info about the action command */
- lol_add( frame->args, list_push_back( list_push_back( list_push_back( list_new(
- outf_time( &time->start ) ),
- outf_time( &time->end ) ),
- outf_double( time->user ) ),
- outf_double( time->system ) ) );
-
- /* Call the rule. */
- evaluate_rule( bindrule( rulename , root_module() ), rulename, frame );
-
- /* Clean up. */
- frame_free( frame );
- }
-}
-
-
-/*
- * call_action_rule() - Look up the __ACTION_RULE__ variable on the given
- * target, and if non-empty, invoke the rule it names, passing the given info,
- * timing_info, executed command and command output.
- */
-
-static void call_action_rule
-(
- TARGET * target,
- int status,
- timing_info const * time,
- char const * executed_command,
- char const * command_output
-)
-{
- LIST * action_rule;
-
- pushsettings( root_module(), target->settings );
- action_rule = var_get( root_module(), constant_ACTION_RULE );
- popsettings( root_module(), target->settings );
-
- if ( !list_empty( action_rule ) )
- {
- /* rule action-rule (
- args * :
- target :
- command status start end user system :
- output ? ) */
-
- /* Prepare the argument list. */
- FRAME frame[ 1 ];
- OBJECT * rulename = list_front( action_rule );
- frame_init( frame );
-
- /* args * :: $(__ACTION_RULE__[2-]) */
- lol_add( frame->args, list_copy_range( action_rule, list_next(
- list_begin( action_rule ) ), list_end( action_rule ) ) );
-
- /* target :: the name of the target */
- lol_add( frame->args, list_new( object_copy( target->name ) ) );
-
- /* command status start end user system :: info about the action command
- */
- lol_add( frame->args,
- list_push_back( list_push_back( list_push_back( list_push_back( list_push_back( list_new(
- object_new( executed_command ) ),
- outf_int( status ) ),
- outf_time( &time->start ) ),
- outf_time( &time->end ) ),
- outf_double( time->user ) ),
- outf_double( time->system ) ) );
-
- /* output ? :: the output of the action command */
- if ( command_output )
- lol_add( frame->args, list_new( object_new( command_output ) ) );
- else
- lol_add( frame->args, L0 );
-
- /* Call the rule. */
- evaluate_rule( bindrule( rulename, root_module() ), rulename, frame );
-
- /* Clean up. */
- frame_free( frame );
- }
-}
-
-
-/*
- * make1c_closure() - handle command execution completion and go to MAKE1C.
- *
- * Internal function passed as a notification callback for when a command
- * finishes getting executed by the OS or called directly when faking that a
- * command had been executed by the OS.
- *
- * Now all we need to do is fiddle with the command exit status and push a new
- * MAKE1C state to execute the next command scheduled for building this target
- * or close up the target's build process in case there are no more commands
- * scheduled for it. On interrupts, we bail heavily.
- */
-
-static void make1c_closure
-(
- void * const closure,
- int status_orig,
- timing_info const * const time,
- char const * const cmd_stdout,
- char const * const cmd_stderr,
- int const cmd_exit_reason
-)
-{
- TARGET * const t = (TARGET *)closure;
- CMD * const cmd = (CMD *)t->cmds;
- char const * rule_name = 0;
- char const * target_name = 0;
-
- assert( cmd );
-
- --cmdsrunning;
-
- /* Calculate the target's status from the cmd execution result. */
- {
- /* Store the target's status. */
- t->status = status_orig;
-
- /* Invert OK/FAIL target status when FAIL_EXPECTED has been applied. */
- if ( t->flags & T_FLAG_FAIL_EXPECTED && !globs.noexec )
- {
- switch ( t->status )
- {
- case EXEC_CMD_FAIL: t->status = EXEC_CMD_OK; break;
- case EXEC_CMD_OK: t->status = EXEC_CMD_FAIL; break;
- }
- }
-
- /* Ignore failures for actions marked as 'ignore'. */
- if ( t->status == EXEC_CMD_FAIL && cmd->rule->actions->flags &
- RULE_IGNORE )
- t->status = EXEC_CMD_OK;
- }
-
- if ( DEBUG_MAKEQ ||
- ( DEBUG_MAKE && !( cmd->rule->actions->flags & RULE_QUIETLY ) ) )
- {
- rule_name = object_str( cmd->rule->name );
- target_name = object_str( list_front( lol_get( (LOL *)&cmd->args, 0 ) )
- );
- }
-
- out_action( rule_name, target_name, cmd->buf->value, cmd_stdout, cmd_stderr,
- cmd_exit_reason );
-
- if ( !globs.noexec )
- {
- call_timing_rule( t, time );
- if ( DEBUG_EXECCMD )
- printf( "%f sec system; %f sec user\n", time->system, time->user );
-
- /* Assume -p0 is in effect, i.e. cmd_stdout contains merged output. */
- call_action_rule( t, status_orig, time, cmd->buf->value, cmd_stdout );
- }
-
- /* Print command text on failure. */
- if ( t->status == EXEC_CMD_FAIL && DEBUG_MAKE )
- {
- if ( !DEBUG_EXEC )
- printf( "%s\n", cmd->buf->value );
-
- printf( "...failed %s ", object_str( cmd->rule->name ) );
- list_print( lol_get( (LOL *)&cmd->args, 0 ) );
- printf( "...\n" );
- }
-
- /* On interrupt, set quit so _everything_ fails. Do the same for failed
- * commands if we were asked to stop the build in case of any errors.
- */
- if ( t->status == EXEC_CMD_INTR )
- {
- ++intr;
- ++quit;
- }
- if ( t->status == EXEC_CMD_FAIL && globs.quitquick )
- ++quit;
-
- /* If the command was not successful remove all of its targets not marked as
- * "precious".
- */
- if ( t->status != EXEC_CMD_OK )
- {
- LIST * const targets = lol_get( (LOL *)&cmd->args, 0 );
- LISTITER iter = list_begin( targets );
- LISTITER const end = list_end( targets );
- for ( ; iter != end; iter = list_next( iter ) )
- {
- char const * const filename = object_str( list_item( iter ) );
- TARGET const * const t = bindtarget( list_item( iter ) );
- if ( !( t->flags & T_FLAG_PRECIOUS ) && !unlink( filename ) )
- printf( "...removing %s\n", filename );
- }
- }
-
- /* Free this command and push the MAKE1C state to execute the next one
- * scheduled for building this same target.
- */
- t->cmds = (char *)cmd_next( cmd );
- cmd_free( cmd );
- push_state( &state_stack, t, NULL, T_STATE_MAKE1C );
-}
-
-
-/*
- * swap_settings() - replace the settings from the current module and target
- * with those from the new module and target
- */
-
-static void swap_settings
-(
- module_t * * current_module,
- TARGET * * current_target,
- module_t * new_module,
- TARGET * new_target
-)
-{
- if ( ( new_target == *current_target ) &&
- ( new_module == *current_module ) )
- return;
-
- if ( *current_target )
- popsettings( *current_module, (*current_target)->settings );
-
- if ( new_target )
- pushsettings( new_module, new_target->settings );
-
- *current_module = new_module;
- *current_target = new_target;
-}
-
-
-/*
- * make1cmds() - turn ACTIONS into CMDs, grouping, splitting, etc.
- *
- * Essentially copies a chain of ACTIONs to a chain of CMDs, grouping
- * RULE_TOGETHER actions, splitting RULE_PIECEMEAL actions, and handling
- * RULE_NEWSRCS actions. The result is a chain of CMDs which has already had all
- * of its embedded variable references expanded and can now be executed using
- * exec_cmd().
- */
-
-static CMD * make1cmds( TARGET * t )
-{
- CMD * cmds = 0;
- CMD * * cmds_next = &cmds;
- LIST * shell = L0;
- module_t * settings_module = 0;
- TARGET * settings_target = 0;
- ACTIONS * a0;
- int const running_flag = globs.noexec ? A_RUNNING_NOEXEC : A_RUNNING;
-
- /* Step through actions. Actions may be shared with other targets or grouped
- * using RULE_TOGETHER, so actions already seen are skipped.
- */
- for ( a0 = t->actions; a0; a0 = a0->next )
- {
- RULE * rule = a0->action->rule;
- rule_actions * actions = rule->actions;
- SETTINGS * boundvars;
- LIST * nt;
- LIST * ns;
- ACTIONS * a1;
-
- /* Only do rules with commands to execute. If this action has already
- * been executed, use saved status.
- */
- if ( !actions || a0->action->running >= running_flag )
- continue;
-
- a0->action->running = running_flag;
-
- /* Make LISTS of targets and sources. If `execute together` has been
- * specified for this rule, tack on sources from each instance of this
- * rule for this target.
- */
- nt = make1list( L0, a0->action->targets, 0 );
- ns = make1list( L0, a0->action->sources, actions->flags );
- if ( actions->flags & RULE_TOGETHER )
- for ( a1 = a0->next; a1; a1 = a1->next )
- if ( a1->action->rule == rule &&
- a1->action->running < running_flag )
- {
- ns = make1list( ns, a1->action->sources, actions->flags );
- a1->action->running = running_flag;
- }
-
- /* If doing only updated (or existing) sources, but none have been
- * updated (or exist), skip this action.
- */
- if ( list_empty( ns ) &&
- ( actions->flags & ( RULE_NEWSRCS | RULE_EXISTING ) ) )
- {
- list_free( nt );
- continue;
- }
-
- swap_settings( &settings_module, &settings_target, rule->module, t );
- if ( list_empty( shell ) )
- {
- /* shell is per-target */
- shell = var_get( rule->module, constant_JAMSHELL );
- }
-
- /* If we had 'actions xxx bind vars' we bind the vars now. */
- boundvars = make1settings( rule->module, actions->bindlist );
- pushsettings( rule->module, boundvars );
-
- /*
- * Build command, starting with all source args.
- *
- * For actions that allow PIECEMEAL commands, if the constructed command
- * string is too long, we retry constructing it with a reduced number of
- * source arguments presented.
- *
- * While reducing slowly takes a bit of compute time to get things just
- * right, it is worth it to get as close to maximum allowed command
- * string length as possible, because launching the commands we are
- * executing is likely to be much more compute intensive.
- *
- * Note that we loop through at least once, for sourceless actions.
- */
- {
- int const length = list_length( ns );
- int start = 0;
- int chunk = length;
- LIST * cmd_targets = L0;
- LIST * cmd_shell = L0;
- do
- {
- CMD * cmd;
- int cmd_check_result;
- int cmd_error_length;
- int cmd_error_max_length;
- int retry = 0;
- int accept_command = 0;
-
- /* Build cmd: cmd_new() takes ownership of its lists. */
- if ( list_empty( cmd_targets ) ) cmd_targets = list_copy( nt );
- if ( list_empty( cmd_shell ) ) cmd_shell = list_copy( shell );
- cmd = cmd_new( rule, cmd_targets, list_sublist( ns, start,
- chunk ), cmd_shell );
-
- cmd_check_result = exec_check( cmd->buf, &cmd->shell,
- &cmd_error_length, &cmd_error_max_length );
-
- if ( cmd_check_result == EXEC_CHECK_OK )
- {
- accept_command = 1;
- }
- else if ( cmd_check_result == EXEC_CHECK_NOOP )
- {
- accept_command = 1;
- cmd->noop = 1;
- }
- else if ( ( actions->flags & RULE_PIECEMEAL ) && ( chunk > 1 ) )
- {
- /* Too long but splittable. Reduce chunk size slowly and
- * retry.
- */
- assert( cmd_check_result == EXEC_CHECK_TOO_LONG ||
- cmd_check_result == EXEC_CHECK_LINE_TOO_LONG );
- chunk = chunk * 9 / 10;
- retry = 1;
- }
- else
- {
- /* Too long and not splittable. */
- char const * const error_message = cmd_check_result ==
- EXEC_CHECK_TOO_LONG
- ? "is too long"
- : "contains a line that is too long";
- assert( cmd_check_result == EXEC_CHECK_TOO_LONG ||
- cmd_check_result == EXEC_CHECK_LINE_TOO_LONG );
- printf( "%s action %s (%d, max %d):\n", object_str(
- rule->name ), error_message, cmd_error_length,
- cmd_error_max_length );
-
- /* Tell the user what did not fit. */
- fputs( cmd->buf->value, stdout );
- exit( EXITBAD );
- }
-
- assert( !retry || !accept_command );
-
- if ( accept_command )
- {
- /* Chain it up. */
- *cmds_next = cmd;
- cmds_next = &cmd->next;
-
- /* Mark lists we need recreated for the next command since
- * they got consumed by the cmd object.
- */
- cmd_targets = L0;
- cmd_shell = L0;
- }
- else
- {
- /* We can reuse targets & shell lists for the next command
- * if we do not let them die with this cmd object.
- */
- cmd_release_targets_and_shell( cmd );
- cmd_free( cmd );
- }
-
- if ( !retry )
- start += chunk;
- }
- while ( start < length );
- }
-
- /* These were always copied when used. */
- list_free( nt );
- list_free( ns );
-
- /* Free variables with values bound by 'actions xxx bind vars'. */
- popsettings( rule->module, boundvars );
- freesettings( boundvars );
- }
-
- swap_settings( &settings_module, &settings_target, 0, 0 );
- return cmds;
-}
-
-
-/*
- * make1list() - turn a list of targets into a LIST, for $(<) and $(>)
- */
-
-static LIST * make1list( LIST * l, TARGETS * targets, int flags )
-{
- for ( ; targets; targets = targets->next )
- {
- TARGET * t = targets->target;
-
- if ( t->binding == T_BIND_UNBOUND )
- make1bind( t );
-
- if ( ( flags & RULE_EXISTING ) && ( flags & RULE_NEWSRCS ) )
- {
- if ( ( t->binding != T_BIND_EXISTS ) &&
- ( t->fate <= T_FATE_STABLE ) )
- continue;
- }
- else if ( flags & RULE_EXISTING )
- {
- if ( t->binding != T_BIND_EXISTS )
- continue;
- }
- else if ( flags & RULE_NEWSRCS )
- {
- if ( t->fate <= T_FATE_STABLE )
- continue;
- }
-
- /* Prohibit duplicates for RULE_TOGETHER. */
- if ( flags & RULE_TOGETHER )
- {
- LISTITER iter = list_begin( l );
- LISTITER const end = list_end( l );
- for ( ; iter != end; iter = list_next( iter ) )
- if ( object_equal( list_item( iter ), t->boundname ) )
- break;
- if ( iter != end )
- continue;
- }
-
- /* Build new list. */
- l = list_push_back( l, object_copy( t->boundname ) );
- }
-
- return l;
-}
-
-
-/*
- * make1settings() - for vars with bound values, build up replacement lists
- */
-
-static SETTINGS * make1settings( struct module_t * module, LIST * vars )
-{
- SETTINGS * settings = 0;
-
- LISTITER vars_iter = list_begin( vars );
- LISTITER const vars_end = list_end( vars );
- for ( ; vars_iter != vars_end; vars_iter = list_next( vars_iter ) )
- {
- LIST * const l = var_get( module, list_item( vars_iter ) );
- LIST * nl = L0;
- LISTITER iter = list_begin( l );
- LISTITER const end = list_end( l );
-
- for ( ; iter != end; iter = list_next( iter ) )
- {
- TARGET * const t = bindtarget( list_item( iter ) );
-
- /* Make sure the target is bound. */
- if ( t->binding == T_BIND_UNBOUND )
- make1bind( t );
-
- /* Build a new list. */
- nl = list_push_back( nl, object_copy( t->boundname ) );
- }
-
- /* Add to settings chain. */
- settings = addsettings( settings, VAR_SET, list_item( vars_iter ), nl );
- }
-
- return settings;
-}
-
-
-/*
- * make1bind() - bind targets that were not bound during dependency analysis
- *
- * Spot the kludge! If a target is not in the dependency tree, it did not get
- * bound by make0(), so we have to do it here. Ugly.
- */
-
-static void make1bind( TARGET * t )
-{
- if ( t->flags & T_FLAG_NOTFILE )
- return;
-
- pushsettings( root_module(), t->settings );
- object_free( t->boundname );
- t->boundname = search( t->name, &t->time, 0, t->flags & T_FLAG_ISFILE );
- t->binding = timestamp_empty( &t->time ) ? T_BIND_MISSING : T_BIND_EXISTS;
- popsettings( root_module(), t->settings );
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/md5.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/md5.c b/ext/kenlm/jam-files/engine/md5.c
deleted file mode 100644
index c35d96c..0000000
--- a/ext/kenlm/jam-files/engine/md5.c
+++ /dev/null
@@ -1,381 +0,0 @@
-/*
- Copyright (C) 1999, 2000, 2002 Aladdin Enterprises. All rights reserved.
-
- This software is provided 'as-is', without any express or implied
- warranty. In no event will the authors be held liable for any damages
- arising from the use of this software.
-
- Permission is granted to anyone to use this software for any purpose,
- including commercial applications, and to alter it and redistribute it
- freely, subject to the following restrictions:
-
- 1. The origin of this software must not be misrepresented; you must not
- claim that you wrote the original software. If you use this software
- in a product, an acknowledgment in the product documentation would be
- appreciated but is not required.
- 2. Altered source versions must be plainly marked as such, and must not be
- misrepresented as being the original software.
- 3. This notice may not be removed or altered from any source distribution.
-
- L. Peter Deutsch
- ghost@aladdin.com
-
- */
-/* $Id: md5.c,v 1.6 2002/04/13 19:20:28 lpd Exp $ */
-/*
- Independent implementation of MD5 (RFC 1321).
-
- This code implements the MD5 Algorithm defined in RFC 1321, whose
- text is available at
- http://www.ietf.org/rfc/rfc1321.txt
- The code is derived from the text of the RFC, including the test suite
- (section A.5) but excluding the rest of Appendix A. It does not include
- any code or documentation that is identified in the RFC as being
- copyrighted.
-
- The original and principal author of md5.c is L. Peter Deutsch
- <gh...@aladdin.com>. Other authors are noted in the change history
- that follows (in reverse chronological order):
-
- 2002-04-13 lpd Clarified derivation from RFC 1321; now handles byte order
- either statically or dynamically; added missing #include <string.h>
- in library.
- 2002-03-11 lpd Corrected argument list for main(), and added int return
- type, in test program and T value program.
- 2002-02-21 lpd Added missing #include <stdio.h> in test program.
- 2000-07-03 lpd Patched to eliminate warnings about "constant is
- unsigned in ANSI C, signed in traditional"; made test program
- self-checking.
- 1999-11-04 lpd Edited comments slightly for automatic TOC extraction.
- 1999-10-18 lpd Fixed typo in header comment (ansi2knr rather than md5).
- 1999-05-03 lpd Original version.
- */
-
-#include "md5.h"
-#include <string.h>
-
-#undef BYTE_ORDER /* 1 = big-endian, -1 = little-endian, 0 = unknown */
-#ifdef ARCH_IS_BIG_ENDIAN
-# define BYTE_ORDER (ARCH_IS_BIG_ENDIAN ? 1 : -1)
-#else
-# define BYTE_ORDER 0
-#endif
-
-#define T_MASK ((md5_word_t)~0)
-#define T1 /* 0xd76aa478 */ (T_MASK ^ 0x28955b87)
-#define T2 /* 0xe8c7b756 */ (T_MASK ^ 0x173848a9)
-#define T3 0x242070db
-#define T4 /* 0xc1bdceee */ (T_MASK ^ 0x3e423111)
-#define T5 /* 0xf57c0faf */ (T_MASK ^ 0x0a83f050)
-#define T6 0x4787c62a
-#define T7 /* 0xa8304613 */ (T_MASK ^ 0x57cfb9ec)
-#define T8 /* 0xfd469501 */ (T_MASK ^ 0x02b96afe)
-#define T9 0x698098d8
-#define T10 /* 0x8b44f7af */ (T_MASK ^ 0x74bb0850)
-#define T11 /* 0xffff5bb1 */ (T_MASK ^ 0x0000a44e)
-#define T12 /* 0x895cd7be */ (T_MASK ^ 0x76a32841)
-#define T13 0x6b901122
-#define T14 /* 0xfd987193 */ (T_MASK ^ 0x02678e6c)
-#define T15 /* 0xa679438e */ (T_MASK ^ 0x5986bc71)
-#define T16 0x49b40821
-#define T17 /* 0xf61e2562 */ (T_MASK ^ 0x09e1da9d)
-#define T18 /* 0xc040b340 */ (T_MASK ^ 0x3fbf4cbf)
-#define T19 0x265e5a51
-#define T20 /* 0xe9b6c7aa */ (T_MASK ^ 0x16493855)
-#define T21 /* 0xd62f105d */ (T_MASK ^ 0x29d0efa2)
-#define T22 0x02441453
-#define T23 /* 0xd8a1e681 */ (T_MASK ^ 0x275e197e)
-#define T24 /* 0xe7d3fbc8 */ (T_MASK ^ 0x182c0437)
-#define T25 0x21e1cde6
-#define T26 /* 0xc33707d6 */ (T_MASK ^ 0x3cc8f829)
-#define T27 /* 0xf4d50d87 */ (T_MASK ^ 0x0b2af278)
-#define T28 0x455a14ed
-#define T29 /* 0xa9e3e905 */ (T_MASK ^ 0x561c16fa)
-#define T30 /* 0xfcefa3f8 */ (T_MASK ^ 0x03105c07)
-#define T31 0x676f02d9
-#define T32 /* 0x8d2a4c8a */ (T_MASK ^ 0x72d5b375)
-#define T33 /* 0xfffa3942 */ (T_MASK ^ 0x0005c6bd)
-#define T34 /* 0x8771f681 */ (T_MASK ^ 0x788e097e)
-#define T35 0x6d9d6122
-#define T36 /* 0xfde5380c */ (T_MASK ^ 0x021ac7f3)
-#define T37 /* 0xa4beea44 */ (T_MASK ^ 0x5b4115bb)
-#define T38 0x4bdecfa9
-#define T39 /* 0xf6bb4b60 */ (T_MASK ^ 0x0944b49f)
-#define T40 /* 0xbebfbc70 */ (T_MASK ^ 0x4140438f)
-#define T41 0x289b7ec6
-#define T42 /* 0xeaa127fa */ (T_MASK ^ 0x155ed805)
-#define T43 /* 0xd4ef3085 */ (T_MASK ^ 0x2b10cf7a)
-#define T44 0x04881d05
-#define T45 /* 0xd9d4d039 */ (T_MASK ^ 0x262b2fc6)
-#define T46 /* 0xe6db99e5 */ (T_MASK ^ 0x1924661a)
-#define T47 0x1fa27cf8
-#define T48 /* 0xc4ac5665 */ (T_MASK ^ 0x3b53a99a)
-#define T49 /* 0xf4292244 */ (T_MASK ^ 0x0bd6ddbb)
-#define T50 0x432aff97
-#define T51 /* 0xab9423a7 */ (T_MASK ^ 0x546bdc58)
-#define T52 /* 0xfc93a039 */ (T_MASK ^ 0x036c5fc6)
-#define T53 0x655b59c3
-#define T54 /* 0x8f0ccc92 */ (T_MASK ^ 0x70f3336d)
-#define T55 /* 0xffeff47d */ (T_MASK ^ 0x00100b82)
-#define T56 /* 0x85845dd1 */ (T_MASK ^ 0x7a7ba22e)
-#define T57 0x6fa87e4f
-#define T58 /* 0xfe2ce6e0 */ (T_MASK ^ 0x01d3191f)
-#define T59 /* 0xa3014314 */ (T_MASK ^ 0x5cfebceb)
-#define T60 0x4e0811a1
-#define T61 /* 0xf7537e82 */ (T_MASK ^ 0x08ac817d)
-#define T62 /* 0xbd3af235 */ (T_MASK ^ 0x42c50dca)
-#define T63 0x2ad7d2bb
-#define T64 /* 0xeb86d391 */ (T_MASK ^ 0x14792c6e)
-
-
-static void
-md5_process(md5_state_t *pms, const md5_byte_t *data /*[64]*/)
-{
- md5_word_t
- a = pms->abcd[0], b = pms->abcd[1],
- c = pms->abcd[2], d = pms->abcd[3];
- md5_word_t t;
-#if BYTE_ORDER > 0
- /* Define storage only for big-endian CPUs. */
- md5_word_t X[16];
-#else
- /* Define storage for little-endian or both types of CPUs. */
- md5_word_t xbuf[16];
- const md5_word_t *X;
-#endif
-
- {
-#if BYTE_ORDER == 0
- /*
- * Determine dynamically whether this is a big-endian or
- * little-endian machine, since we can use a more efficient
- * algorithm on the latter.
- */
- static const int w = 1;
-
- if (*((const md5_byte_t *)&w)) /* dynamic little-endian */
-#endif
-#if BYTE_ORDER <= 0 /* little-endian */
- {
- /*
- * On little-endian machines, we can process properly aligned
- * data without copying it.
- */
- if (!((data - (const md5_byte_t *)0) & 3)) {
- /* data are properly aligned */
- X = (const md5_word_t *)data;
- } else {
- /* not aligned */
- memcpy(xbuf, data, 64);
- X = xbuf;
- }
- }
-#endif
-#if BYTE_ORDER == 0
- else /* dynamic big-endian */
-#endif
-#if BYTE_ORDER >= 0 /* big-endian */
- {
- /*
- * On big-endian machines, we must arrange the bytes in the
- * right order.
- */
- const md5_byte_t *xp = data;
- int i;
-
-# if BYTE_ORDER == 0
- X = xbuf; /* (dynamic only) */
-# else
-# define xbuf X /* (static only) */
-# endif
- for (i = 0; i < 16; ++i, xp += 4)
- xbuf[i] = xp[0] + (xp[1] << 8) + (xp[2] << 16) + (xp[3] << 24);
- }
-#endif
- }
-
-#define ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32 - (n))))
-
- /* Round 1. */
- /* Let [abcd k s i] denote the operation
- a = b + ((a + F(b,c,d) + X[k] + T[i]) <<< s). */
-#define F(x, y, z) (((x) & (y)) | (~(x) & (z)))
-#define SET(a, b, c, d, k, s, Ti)\
- t = a + F(b,c,d) + X[k] + Ti;\
- a = ROTATE_LEFT(t, s) + b
- /* Do the following 16 operations. */
- SET(a, b, c, d, 0, 7, T1);
- SET(d, a, b, c, 1, 12, T2);
- SET(c, d, a, b, 2, 17, T3);
- SET(b, c, d, a, 3, 22, T4);
- SET(a, b, c, d, 4, 7, T5);
- SET(d, a, b, c, 5, 12, T6);
- SET(c, d, a, b, 6, 17, T7);
- SET(b, c, d, a, 7, 22, T8);
- SET(a, b, c, d, 8, 7, T9);
- SET(d, a, b, c, 9, 12, T10);
- SET(c, d, a, b, 10, 17, T11);
- SET(b, c, d, a, 11, 22, T12);
- SET(a, b, c, d, 12, 7, T13);
- SET(d, a, b, c, 13, 12, T14);
- SET(c, d, a, b, 14, 17, T15);
- SET(b, c, d, a, 15, 22, T16);
-#undef SET
-
- /* Round 2. */
- /* Let [abcd k s i] denote the operation
- a = b + ((a + G(b,c,d) + X[k] + T[i]) <<< s). */
-#define G(x, y, z) (((x) & (z)) | ((y) & ~(z)))
-#define SET(a, b, c, d, k, s, Ti)\
- t = a + G(b,c,d) + X[k] + Ti;\
- a = ROTATE_LEFT(t, s) + b
- /* Do the following 16 operations. */
- SET(a, b, c, d, 1, 5, T17);
- SET(d, a, b, c, 6, 9, T18);
- SET(c, d, a, b, 11, 14, T19);
- SET(b, c, d, a, 0, 20, T20);
- SET(a, b, c, d, 5, 5, T21);
- SET(d, a, b, c, 10, 9, T22);
- SET(c, d, a, b, 15, 14, T23);
- SET(b, c, d, a, 4, 20, T24);
- SET(a, b, c, d, 9, 5, T25);
- SET(d, a, b, c, 14, 9, T26);
- SET(c, d, a, b, 3, 14, T27);
- SET(b, c, d, a, 8, 20, T28);
- SET(a, b, c, d, 13, 5, T29);
- SET(d, a, b, c, 2, 9, T30);
- SET(c, d, a, b, 7, 14, T31);
- SET(b, c, d, a, 12, 20, T32);
-#undef SET
-
- /* Round 3. */
- /* Let [abcd k s t] denote the operation
- a = b + ((a + H(b,c,d) + X[k] + T[i]) <<< s). */
-#define H(x, y, z) ((x) ^ (y) ^ (z))
-#define SET(a, b, c, d, k, s, Ti)\
- t = a + H(b,c,d) + X[k] + Ti;\
- a = ROTATE_LEFT(t, s) + b
- /* Do the following 16 operations. */
- SET(a, b, c, d, 5, 4, T33);
- SET(d, a, b, c, 8, 11, T34);
- SET(c, d, a, b, 11, 16, T35);
- SET(b, c, d, a, 14, 23, T36);
- SET(a, b, c, d, 1, 4, T37);
- SET(d, a, b, c, 4, 11, T38);
- SET(c, d, a, b, 7, 16, T39);
- SET(b, c, d, a, 10, 23, T40);
- SET(a, b, c, d, 13, 4, T41);
- SET(d, a, b, c, 0, 11, T42);
- SET(c, d, a, b, 3, 16, T43);
- SET(b, c, d, a, 6, 23, T44);
- SET(a, b, c, d, 9, 4, T45);
- SET(d, a, b, c, 12, 11, T46);
- SET(c, d, a, b, 15, 16, T47);
- SET(b, c, d, a, 2, 23, T48);
-#undef SET
-
- /* Round 4. */
- /* Let [abcd k s t] denote the operation
- a = b + ((a + I(b,c,d) + X[k] + T[i]) <<< s). */
-#define I(x, y, z) ((y) ^ ((x) | ~(z)))
-#define SET(a, b, c, d, k, s, Ti)\
- t = a + I(b,c,d) + X[k] + Ti;\
- a = ROTATE_LEFT(t, s) + b
- /* Do the following 16 operations. */
- SET(a, b, c, d, 0, 6, T49);
- SET(d, a, b, c, 7, 10, T50);
- SET(c, d, a, b, 14, 15, T51);
- SET(b, c, d, a, 5, 21, T52);
- SET(a, b, c, d, 12, 6, T53);
- SET(d, a, b, c, 3, 10, T54);
- SET(c, d, a, b, 10, 15, T55);
- SET(b, c, d, a, 1, 21, T56);
- SET(a, b, c, d, 8, 6, T57);
- SET(d, a, b, c, 15, 10, T58);
- SET(c, d, a, b, 6, 15, T59);
- SET(b, c, d, a, 13, 21, T60);
- SET(a, b, c, d, 4, 6, T61);
- SET(d, a, b, c, 11, 10, T62);
- SET(c, d, a, b, 2, 15, T63);
- SET(b, c, d, a, 9, 21, T64);
-#undef SET
-
- /* Then perform the following additions. (That is increment each
- of the four registers by the value it had before this block
- was started.) */
- pms->abcd[0] += a;
- pms->abcd[1] += b;
- pms->abcd[2] += c;
- pms->abcd[3] += d;
-}
-
-void
-md5_init(md5_state_t *pms)
-{
- pms->count[0] = pms->count[1] = 0;
- pms->abcd[0] = 0x67452301;
- pms->abcd[1] = /*0xefcdab89*/ T_MASK ^ 0x10325476;
- pms->abcd[2] = /*0x98badcfe*/ T_MASK ^ 0x67452301;
- pms->abcd[3] = 0x10325476;
-}
-
-void
-md5_append(md5_state_t *pms, const md5_byte_t *data, int nbytes)
-{
- const md5_byte_t *p = data;
- int left = nbytes;
- int offset = (pms->count[0] >> 3) & 63;
- md5_word_t nbits = (md5_word_t)(nbytes << 3);
-
- if (nbytes <= 0)
- return;
-
- /* Update the message length. */
- pms->count[1] += nbytes >> 29;
- pms->count[0] += nbits;
- if (pms->count[0] < nbits)
- pms->count[1]++;
-
- /* Process an initial partial block. */
- if (offset) {
- int copy = (offset + nbytes > 64 ? 64 - offset : nbytes);
-
- memcpy(pms->buf + offset, p, copy);
- if (offset + copy < 64)
- return;
- p += copy;
- left -= copy;
- md5_process(pms, pms->buf);
- }
-
- /* Process full blocks. */
- for (; left >= 64; p += 64, left -= 64)
- md5_process(pms, p);
-
- /* Process a final partial block. */
- if (left)
- memcpy(pms->buf, p, left);
-}
-
-void
-md5_finish(md5_state_t *pms, md5_byte_t digest[16])
-{
- static const md5_byte_t pad[64] = {
- 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
- };
- md5_byte_t data[8];
- int i;
-
- /* Save the length before padding. */
- for (i = 0; i < 8; ++i)
- data[i] = (md5_byte_t)(pms->count[i >> 2] >> ((i & 3) << 3));
- /* Pad to 56 bytes mod 64. */
- md5_append(pms, pad, ((55 - (pms->count[0] >> 3)) & 63) + 1);
- /* Append the length. */
- md5_append(pms, data, 8);
- for (i = 0; i < 16; ++i)
- digest[i] = (md5_byte_t)(pms->abcd[i >> 2] >> ((i & 3) << 3));
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/md5.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/md5.h b/ext/kenlm/jam-files/engine/md5.h
deleted file mode 100644
index 698c995..0000000
--- a/ext/kenlm/jam-files/engine/md5.h
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- Copyright (C) 1999, 2002 Aladdin Enterprises. All rights reserved.
-
- This software is provided 'as-is', without any express or implied
- warranty. In no event will the authors be held liable for any damages
- arising from the use of this software.
-
- Permission is granted to anyone to use this software for any purpose,
- including commercial applications, and to alter it and redistribute it
- freely, subject to the following restrictions:
-
- 1. The origin of this software must not be misrepresented; you must not
- claim that you wrote the original software. If you use this software
- in a product, an acknowledgment in the product documentation would be
- appreciated but is not required.
- 2. Altered source versions must be plainly marked as such, and must not be
- misrepresented as being the original software.
- 3. This notice may not be removed or altered from any source distribution.
-
- L. Peter Deutsch
- ghost@aladdin.com
-
- */
-/* $Id: md5.h,v 1.4 2002/04/13 19:20:28 lpd Exp $ */
-/*
- Independent implementation of MD5 (RFC 1321).
-
- This code implements the MD5 Algorithm defined in RFC 1321, whose
- text is available at
- http://www.ietf.org/rfc/rfc1321.txt
- The code is derived from the text of the RFC, including the test suite
- (section A.5) but excluding the rest of Appendix A. It does not include
- any code or documentation that is identified in the RFC as being
- copyrighted.
-
- The original and principal author of md5.h is L. Peter Deutsch
- <gh...@aladdin.com>. Other authors are noted in the change history
- that follows (in reverse chronological order):
-
- 2002-04-13 lpd Removed support for non-ANSI compilers; removed
- references to Ghostscript; clarified derivation from RFC 1321;
- now handles byte order either statically or dynamically.
- 1999-11-04 lpd Edited comments slightly for automatic TOC extraction.
- 1999-10-18 lpd Fixed typo in header comment (ansi2knr rather than md5);
- added conditionalization for C++ compilation from Martin
- Purschke <pu...@bnl.gov>.
- 1999-05-03 lpd Original version.
- */
-
-#ifndef md5_INCLUDED
-# define md5_INCLUDED
-
-/*
- * This package supports both compile-time and run-time determination of CPU
- * byte order. If ARCH_IS_BIG_ENDIAN is defined as 0, the code will be
- * compiled to run only on little-endian CPUs; if ARCH_IS_BIG_ENDIAN is
- * defined as non-zero, the code will be compiled to run only on big-endian
- * CPUs; if ARCH_IS_BIG_ENDIAN is not defined, the code will be compiled to
- * run on either big- or little-endian CPUs, but will run slightly less
- * efficiently on either one than if ARCH_IS_BIG_ENDIAN is defined.
- */
-
-typedef unsigned char md5_byte_t; /* 8-bit byte */
-typedef unsigned int md5_word_t; /* 32-bit word */
-
-/* Define the state of the MD5 Algorithm. */
-typedef struct md5_state_s {
- md5_word_t count[2]; /* message length in bits, lsw first */
- md5_word_t abcd[4]; /* digest buffer */
- md5_byte_t buf[64]; /* accumulate block */
-} md5_state_t;
-
-#ifdef __cplusplus
-extern "C"
-{
-#endif
-
-/* Initialize the algorithm. */
-void md5_init(md5_state_t *pms);
-
-/* Append a string to the message. */
-void md5_append(md5_state_t *pms, const md5_byte_t *data, int nbytes);
-
-/* Finish the message and return the digest. */
-void md5_finish(md5_state_t *pms, md5_byte_t digest[16]);
-
-#ifdef __cplusplus
-} /* end extern "C" */
-#endif
-
-#endif /* md5_INCLUDED */
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/mem.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/mem.c b/ext/kenlm/jam-files/engine/mem.c
deleted file mode 100644
index 6a11fb3..0000000
--- a/ext/kenlm/jam-files/engine/mem.c
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
-Copyright Rene Rivera 2006.
-Distributed under the Boost Software License, Version 1.0.
-(See accompanying file LICENSE_1_0.txt or copy at
-http://www.boost.org/LICENSE_1_0.txt)
-*/
-
-#include "jam.h"
-
-#ifdef OPT_BOEHM_GC
-
- /* Compile the Boehm GC as one big chunk of code. It's much easier
- this way, than trying to make radical changes to the bjam build
- scripts. */
-
- #define ATOMIC_UNCOLLECTABLE
- #define NO_EXECUTE_PERMISSION
- #define ALL_INTERIOR_POINTERS
-
- #define LARGE_CONFIG
- /*
- #define NO_SIGNALS
- #define SILENT
- */
- #ifndef GC_DEBUG
- #define NO_DEBUGGING
- #endif
-
- #ifdef __GLIBC__
- #define __USE_GNU
- #endif
-
- #include "boehm_gc/reclaim.c"
- #include "boehm_gc/allchblk.c"
- #include "boehm_gc/misc.c"
- #include "boehm_gc/alloc.c"
- #include "boehm_gc/mach_dep.c"
- #include "boehm_gc/os_dep.c"
- #include "boehm_gc/mark_rts.c"
- #include "boehm_gc/headers.c"
- #include "boehm_gc/mark.c"
- #include "boehm_gc/obj_map.c"
- #include "boehm_gc/pcr_interface.c"
- #include "boehm_gc/blacklst.c"
- #include "boehm_gc/new_hblk.c"
- #include "boehm_gc/real_malloc.c"
- #include "boehm_gc/dyn_load.c"
- #include "boehm_gc/dbg_mlc.c"
- #include "boehm_gc/malloc.c"
- #include "boehm_gc/stubborn.c"
- #include "boehm_gc/checksums.c"
- #include "boehm_gc/pthread_support.c"
- #include "boehm_gc/pthread_stop_world.c"
- #include "boehm_gc/darwin_stop_world.c"
- #include "boehm_gc/typd_mlc.c"
- #include "boehm_gc/ptr_chck.c"
- #include "boehm_gc/mallocx.c"
- #include "boehm_gc/gcj_mlc.c"
- #include "boehm_gc/specific.c"
- #include "boehm_gc/gc_dlopen.c"
- #include "boehm_gc/backgraph.c"
- #include "boehm_gc/win32_threads.c"
-
- /* Needs to be last. */
- #include "boehm_gc/finalize.c"
-
-#elif defined(OPT_DUMA)
-
- #ifdef OS_NT
- #define WIN32
- #endif
- #include "duma/duma.c"
- #include "duma/print.c"
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/mem.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/mem.h b/ext/kenlm/jam-files/engine/mem.h
deleted file mode 100644
index 8718b07..0000000
--- a/ext/kenlm/jam-files/engine/mem.h
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Copyright 2006. Rene Rivera
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#ifndef BJAM_MEM_H
-#define BJAM_MEM_H
-
-#ifdef OPT_BOEHM_GC
-
- /* Use Boehm GC memory allocator. */
- #include <gc.h>
-
- #define bjam_malloc_x(s) memset(GC_malloc(s),0,s)
- #define bjam_malloc_atomic_x(s) memset(GC_malloc_atomic(s),0,s)
- #define bjam_calloc_x(n,s) memset(GC_malloc((n)*(s)),0,(n)*(s))
- #define bjam_calloc_atomic_x(n,s) memset(GC_malloc_atomic((n)*(s)),0,(n)*(s))
- #define bjam_realloc_x(p,s) GC_realloc(p,s)
- #define bjam_free_x(p) GC_free(p)
- #define bjam_mem_init_x() GC_init(); GC_enable_incremental()
-
- #define bjam_malloc_raw_x(s) malloc(s)
- #define bjam_calloc_raw_x(n,s) calloc(n,s)
- #define bjam_realloc_raw_x(p,s) realloc(p,s)
- #define bjam_free_raw_x(p) free(p)
-
- #ifndef BJAM_NEWSTR_NO_ALLOCATE
- # define BJAM_NEWSTR_NO_ALLOCATE
- #endif
-
-#elif defined( OPT_DUMA )
-
- /* Use Duma memory debugging library. */
- #include <stdlib.h>
-
- #define _DUMA_CONFIG_H_
- #define DUMA_NO_GLOBAL_MALLOC_FREE
- #define DUMA_EXPLICIT_INIT
- #define DUMA_NO_THREAD_SAFETY
- #define DUMA_NO_CPP_SUPPORT
- /* #define DUMA_NO_LEAKDETECTION */
- /* #define DUMA_USE_FRAMENO */
- /* #define DUMA_PREFER_ATEXIT */
- /* #define DUMA_OLD_DEL_MACRO */
- /* #define DUMA_NO_HANG_MSG */
- #define DUMA_PAGE_SIZE 4096
- #define DUMA_MIN_ALIGNMENT 1
- /* #define DUMA_GNU_INIT_ATTR 0 */
- typedef unsigned int DUMA_ADDR;
- typedef unsigned int DUMA_SIZE;
- #include <duma.h>
-
- #define bjam_malloc_x(s) malloc(s)
- #define bjam_calloc_x(n,s) calloc(n,s)
- #define bjam_realloc_x(p,s) realloc(p,s)
- #define bjam_free_x(p) free(p)
-
- #ifndef BJAM_NEWSTR_NO_ALLOCATE
- # define BJAM_NEWSTR_NO_ALLOCATE
- #endif
-
-#else
-
- /* Standard C memory allocation. */
- #include <stdlib.h>
-
- #define bjam_malloc_x(s) malloc(s)
- #define bjam_calloc_x(n,s) calloc(n,s)
- #define bjam_realloc_x(p,s) realloc(p,s)
- #define bjam_free_x(p) free(p)
-
-#endif
-
-#ifndef bjam_malloc_atomic_x
- #define bjam_malloc_atomic_x(s) bjam_malloc_x(s)
-#endif
-#ifndef bjam_calloc_atomic_x
- #define bjam_calloc_atomic_x(n,s) bjam_calloc_x(n,s)
-#endif
-#ifndef bjam_mem_init_x
- #define bjam_mem_init_x()
-#endif
-#ifndef bjam_mem_close_x
- #define bjam_mem_close_x()
-#endif
-#ifndef bjam_malloc_raw_x
- #define bjam_malloc_raw_x(s) bjam_malloc_x(s)
-#endif
-#ifndef bjam_calloc_raw_x
- #define bjam_calloc_raw_x(n,s) bjam_calloc_x(n,s)
-#endif
-#ifndef bjam_realloc_raw_x
- #define bjam_realloc_raw_x(p,s) bjam_realloc_x(p,s)
-#endif
-#ifndef bjam_free_raw_x
- #define bjam_free_raw_x(p) bjam_free_x(p)
-#endif
-
-#ifdef OPT_DEBUG_PROFILE
- /* Profile tracing of memory allocations. */
- #include "debug.h"
-
- #define BJAM_MALLOC(s) (profile_memory(s), bjam_malloc_x(s))
- #define BJAM_MALLOC_ATOMIC(s) (profile_memory(s), bjam_malloc_atomic_x(s))
- #define BJAM_CALLOC(n,s) (profile_memory(n*s), bjam_calloc_x(n,s))
- #define BJAM_CALLOC_ATOMIC(n,s) (profile_memory(n*s), bjam_calloc_atomic_x(n,s))
- #define BJAM_REALLOC(p,s) (profile_memory(s), bjam_realloc_x(p,s))
-
- #define BJAM_MALLOC_RAW(s) (profile_memory(s), bjam_malloc_raw_x(s))
- #define BJAM_CALLOC_RAW(n,s) (profile_memory(n*s), bjam_calloc_raw_x(n,s))
- #define BJAM_REALLOC_RAW(p,s) (profile_memory(s), bjam_realloc_raw_x(p,s))
-#else
- /* No mem tracing. */
- #define BJAM_MALLOC(s) bjam_malloc_x(s)
- #define BJAM_MALLOC_ATOMIC(s) bjam_malloc_atomic_x(s)
- #define BJAM_CALLOC(n,s) bjam_calloc_x(n,s)
- #define BJAM_CALLOC_ATOMIC(n,s) bjam_calloc_atomic_x(n,s)
- #define BJAM_REALLOC(p,s) bjam_realloc_x(p,s)
-
- #define BJAM_MALLOC_RAW(s) bjam_malloc_raw_x(s)
- #define BJAM_CALLOC_RAW(n,s) bjam_calloc_raw_x(n,s)
- #define BJAM_REALLOC_RAW(p,s) bjam_realloc_raw_x(p,s)
-#endif
-
-#define BJAM_MEM_INIT() bjam_mem_init_x()
-#define BJAM_MEM_CLOSE() bjam_mem_close_x()
-
-#define BJAM_FREE(p) bjam_free_x(p)
-#define BJAM_FREE_RAW(p) bjam_free_raw_x(p)
-
-#endif
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/mkjambase.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/mkjambase.c b/ext/kenlm/jam-files/engine/mkjambase.c
deleted file mode 100644
index cdf5998..0000000
--- a/ext/kenlm/jam-files/engine/mkjambase.c
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Copyright 1993, 1995 Christopher Seiwald.
- *
- * This file is part of Jam - see jam.c for Copyright information.
- */
-
-/* This file is ALSO:
- * Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
- */
-
-/*
- * mkjambase.c - turn Jambase into a big C structure
- *
- * Usage: mkjambase jambase.c Jambase ...
- *
- * Results look like this:
- *
- * char *jambase[] = {
- * "...\n",
- * ...
- * 0 };
- *
- * Handles \'s and "'s specially; knows to delete blank and comment lines.
- *
- */
-
-#include <stdio.h>
-#include <string.h>
-
-
-int main( int argc, char * * argv, char * * envp )
-{
- char buf[ 1024 ];
- FILE * fin;
- FILE * fout;
- char * p;
- int doDotC = 0;
-
- if ( argc < 3 )
- {
- fprintf( stderr, "usage: %s jambase.c Jambase ...\n", argv[ 0 ] );
- return -1;
- }
-
- if ( !( fout = fopen( argv[1], "w" ) ) )
- {
- perror( argv[ 1 ] );
- return -1;
- }
-
- /* If the file ends in .c generate a C source file. */
- if ( ( p = strrchr( argv[1], '.' ) ) && !strcmp( p, ".c" ) )
- doDotC++;
-
- /* Now process the files. */
-
- argc -= 2;
- argv += 2;
-
- if ( doDotC )
- {
- fprintf( fout, "/* Generated by mkjambase from Jambase */\n" );
- fprintf( fout, "char *jambase[] = {\n" );
- }
-
- for ( ; argc--; ++argv )
- {
- if ( !( fin = fopen( *argv, "r" ) ) )
- {
- perror( *argv );
- return -1;
- }
-
- if ( doDotC )
- fprintf( fout, "/* %s */\n", *argv );
- else
- fprintf( fout, "### %s ###\n", *argv );
-
- while ( fgets( buf, sizeof( buf ), fin ) )
- {
- if ( doDotC )
- {
- char * p = buf;
-
- /* Strip leading whitespace. */
- while ( ( *p == ' ' ) || ( *p == '\t' ) || ( *p == '\n' ) )
- ++p;
-
- /* Drop comments and empty lines. */
- if ( ( *p == '#' ) || !*p )
- continue;
-
- /* Copy. */
- putc( '"', fout );
- for ( ; *p && ( *p != '\n' ); ++p )
- switch ( *p )
- {
- case '\\': putc( '\\', fout ); putc( '\\', fout ); break;
- case '"' : putc( '\\', fout ); putc( '"' , fout ); break;
- case '\r': break;
- default: putc( *p, fout ); break;
- }
-
- fprintf( fout, "\\n\",\n" );
- }
- else
- {
- fprintf( fout, "%s", buf );
- }
- }
-
- fclose( fin );
- }
-
- if ( doDotC )
- fprintf( fout, "0 };\n" );
-
- fclose( fout );
-
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/modules.c
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/modules.c b/ext/kenlm/jam-files/engine/modules.c
deleted file mode 100644
index 6be82fe..0000000
--- a/ext/kenlm/jam-files/engine/modules.c
+++ /dev/null
@@ -1,431 +0,0 @@
-/*
- * Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or copy at
- * http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#include "jam.h"
-#include "modules.h"
-
-#include "hash.h"
-#include "lists.h"
-#include "native.h"
-#include "object.h"
-#include "parse.h"
-#include "rules.h"
-#include "strings.h"
-#include "variable.h"
-
-#include <assert.h>
-#include <string.h>
-
-static struct hash * module_hash = 0;
-static module_t root;
-
-
-module_t * bindmodule( OBJECT * name )
-{
- if ( !name )
- return &root;
-
- {
- PROFILE_ENTER( BINDMODULE );
-
- module_t * m;
- int found;
-
- if ( !module_hash )
- module_hash = hashinit( sizeof( module_t ), "modules" );
-
- m = (module_t *)hash_insert( module_hash, name, &found );
- if ( !found )
- {
- m->name = object_copy( name );
- m->variables = 0;
- m->variable_indices = 0;
- m->num_fixed_variables = 0;
- m->fixed_variables = 0;
- m->rules = 0;
- m->imported_modules = 0;
- m->class_module = 0;
- m->native_rules = 0;
- m->user_module = 0;
- }
-
- PROFILE_EXIT( BINDMODULE );
-
- return m;
- }
-}
-
-
-/*
- * demand_rules() - Get the module's "rules" hash on demand.
- */
-struct hash * demand_rules( module_t * m )
-{
- if ( !m->rules )
- m->rules = hashinit( sizeof( RULE ), "rules" );
- return m->rules;
-}
-
-
-/*
- * delete_module() - wipe out the module's rules and variables.
- */
-
-static void delete_rule_( void * xrule, void * data )
-{
- rule_free( (RULE *)xrule );
-}
-
-
-static void delete_native_rule( void * xrule, void * data )
-{
- native_rule_t * rule = (native_rule_t *)xrule;
- object_free( rule->name );
- if ( rule->procedure )
- function_free( rule->procedure );
-}
-
-
-static void delete_imported_modules( void * xmodule_name, void * data )
-{
- object_free( *(OBJECT * *)xmodule_name );
-}
-
-
-static void free_fixed_variable( void * xvar, void * data );
-
-void delete_module( module_t * m )
-{
- /* Clear out all the rules. */
- if ( m->rules )
- {
- hashenumerate( m->rules, delete_rule_, (void *)0 );
- hash_free( m->rules );
- m->rules = 0;
- }
-
- if ( m->native_rules )
- {
- hashenumerate( m->native_rules, delete_native_rule, (void *)0 );
- hash_free( m->native_rules );
- m->native_rules = 0;
- }
-
- if ( m->variables )
- {
- var_done( m );
- m->variables = 0;
- }
-
- if ( m->fixed_variables )
- {
- int i;
- for ( i = 0; i < m->num_fixed_variables; ++i )
- {
- list_free( m->fixed_variables[ i ] );
- }
- BJAM_FREE( m->fixed_variables );
- m->fixed_variables = 0;
- }
-
- if ( m->variable_indices )
- {
- hashenumerate( m->variable_indices, &free_fixed_variable, (void *)0 );
- hash_free( m->variable_indices );
- m->variable_indices = 0;
- }
-
- if ( m->imported_modules )
- {
- hashenumerate( m->imported_modules, delete_imported_modules, (void *)0 );
- hash_free( m->imported_modules );
- m->imported_modules = 0;
- }
-}
-
-
-struct module_stats
-{
- OBJECT * module_name;
- struct hashstats rules_stats[ 1 ];
- struct hashstats variables_stats[ 1 ];
- struct hashstats variable_indices_stats[ 1 ];
- struct hashstats imported_modules_stats[ 1 ];
-};
-
-
-static void module_stat( struct hash * hp, OBJECT * module, const char * name )
-{
- if ( hp )
- {
- struct hashstats stats[ 1 ];
- string id[ 1 ];
- hashstats_init( stats );
- string_new( id );
- string_append( id, object_str( module ) );
- string_push_back( id, ' ' );
- string_append( id, name );
-
- hashstats_add( stats, hp );
- hashstats_print( stats, id->value );
-
- string_free( id );
- }
-}
-
-
-static void class_module_stat( struct hashstats * stats, OBJECT * module, const char * name )
-{
- if ( stats->item_size )
- {
- string id[ 1 ];
- string_new( id );
- string_append( id, object_str( module ) );
- string_append( id, " object " );
- string_append( id, name );
-
- hashstats_print( stats, id->value );
-
- string_free( id );
- }
-}
-
-
-static void stat_module( void * xmodule, void * data )
-{
- module_t *m = (module_t *)xmodule;
-
- if ( DEBUG_MEM || DEBUG_PROFILE )
- {
- struct hash * class_info = (struct hash *)data;
- if ( m->class_module )
- {
- int found;
- struct module_stats * ms = (struct module_stats *)hash_insert( class_info, m->class_module->name, &found );
- if ( !found )
- {
- ms->module_name = m->class_module->name;
- hashstats_init( ms->rules_stats );
- hashstats_init( ms->variables_stats );
- hashstats_init( ms->variable_indices_stats );
- hashstats_init( ms->imported_modules_stats );
- }
-
- hashstats_add( ms->rules_stats, m->rules );
- hashstats_add( ms->variables_stats, m->variables );
- hashstats_add( ms->variable_indices_stats, m->variable_indices );
- hashstats_add( ms->imported_modules_stats, m->imported_modules );
- }
- else
- {
- module_stat( m->rules, m->name, "rules" );
- module_stat( m->variables, m->name, "variables" );
- module_stat( m->variable_indices, m->name, "fixed variables" );
- module_stat( m->imported_modules, m->name, "imported modules" );
- }
- }
-
- delete_module( m );
- object_free( m->name );
-}
-
-static void print_class_stats( void * xstats, void * data )
-{
- struct module_stats * stats = (struct module_stats *)xstats;
- class_module_stat( stats->rules_stats, stats->module_name, "rules" );
- class_module_stat( stats->variables_stats, stats->module_name, "variables" );
- class_module_stat( stats->variable_indices_stats, stats->module_name, "fixed variables" );
- class_module_stat( stats->imported_modules_stats, stats->module_name, "imported modules" );
-}
-
-
-static void delete_module_( void * xmodule, void * data )
-{
- module_t *m = (module_t *)xmodule;
-
- delete_module( m );
- object_free( m->name );
-}
-
-
-void modules_done()
-{
- if ( DEBUG_MEM || DEBUG_PROFILE )
- {
- struct hash * class_hash = hashinit( sizeof( struct module_stats ), "object info" );
- hashenumerate( module_hash, stat_module, (void *)class_hash );
- hashenumerate( class_hash, print_class_stats, (void *)0 );
- hash_free( class_hash );
- }
- hashenumerate( module_hash, delete_module_, (void *)0 );
- hashdone( module_hash );
- module_hash = 0;
- delete_module( &root );
-}
-
-module_t * root_module()
-{
- return &root;
-}
-
-
-void import_module( LIST * module_names, module_t * target_module )
-{
- PROFILE_ENTER( IMPORT_MODULE );
-
- struct hash * h;
- LISTITER iter;
- LISTITER end;
-
- if ( !target_module->imported_modules )
- target_module->imported_modules = hashinit( sizeof( char * ), "imported"
- );
- h = target_module->imported_modules;
-
- iter = list_begin( module_names );
- end = list_end( module_names );
- for ( ; iter != end; iter = list_next( iter ) )
- {
- int found;
- OBJECT * const s = list_item( iter );
- OBJECT * * const ss = (OBJECT * *)hash_insert( h, s, &found );
- if ( !found )
- *ss = object_copy( s );
- }
-
- PROFILE_EXIT( IMPORT_MODULE );
-}
-
-
-static void add_module_name( void * r_, void * result_ )
-{
- OBJECT * * const r = (OBJECT * *)r_;
- LIST * * const result = (LIST * *)result_;
- *result = list_push_back( *result, object_copy( *r ) );
-}
-
-
-LIST * imported_modules( module_t * module )
-{
- LIST * result = L0;
- if ( module->imported_modules )
- hashenumerate( module->imported_modules, add_module_name, &result );
- return result;
-}
-
-
-FUNCTION * function_bind_variables( FUNCTION *, module_t *, int * counter );
-FUNCTION * function_unbind_variables( FUNCTION * );
-
-struct fixed_variable
-{
- OBJECT * key;
- int n;
-};
-
-struct bind_vars_t
-{
- module_t * module;
- int counter;
-};
-
-
-static void free_fixed_variable( void * xvar, void * data )
-{
- object_free( ( (struct fixed_variable *)xvar )->key );
-}
-
-
-static void bind_variables_for_rule( void * xrule, void * xdata )
-{
- RULE * rule = (RULE *)xrule;
- struct bind_vars_t * data = (struct bind_vars_t *)xdata;
- if ( rule->procedure && rule->module == data->module )
- rule->procedure = function_bind_variables( rule->procedure,
- data->module, &data->counter );
-}
-
-
-void module_bind_variables( struct module_t * m )
-{
- if ( m != root_module() && m->rules )
- {
- struct bind_vars_t data;
- data.module = m;
- data.counter = m->num_fixed_variables;
- hashenumerate( m->rules, &bind_variables_for_rule, &data );
- module_set_fixed_variables( m, data.counter );
- }
-}
-
-
-int module_add_fixed_var( struct module_t * m, OBJECT * name, int * counter )
-{
- struct fixed_variable * v;
- int found;
-
- assert( !m->class_module );
-
- if ( !m->variable_indices )
- m->variable_indices = hashinit( sizeof( struct fixed_variable ), "variable index table" );
-
- v = (struct fixed_variable *)hash_insert( m->variable_indices, name, &found );
- if ( !found )
- {
- v->key = object_copy( name );
- v->n = (*counter)++;
- }
-
- return v->n;
-}
-
-
-LIST * var_get_and_clear_raw( module_t * m, OBJECT * name );
-
-static void load_fixed_variable( void * xvar, void * data )
-{
- struct fixed_variable * var = (struct fixed_variable *)xvar;
- struct module_t * m = (struct module_t *)data;
- if ( var->n >= m->num_fixed_variables )
- m->fixed_variables[ var->n ] = var_get_and_clear_raw( m, var->key );
-}
-
-
-void module_set_fixed_variables( struct module_t * m, int n_variables )
-{
- /* Reallocate */
- struct hash * variable_indices;
- LIST * * fixed_variables = BJAM_MALLOC( n_variables * sizeof( LIST * ) );
- if ( m->fixed_variables )
- {
- memcpy( fixed_variables, m->fixed_variables, m->num_fixed_variables * sizeof( LIST * ) );
- BJAM_FREE( m->fixed_variables );
- }
- m->fixed_variables = fixed_variables;
- variable_indices = m->class_module
- ? m->class_module->variable_indices
- : m->variable_indices;
- if ( variable_indices )
- hashenumerate( variable_indices, &load_fixed_variable, m );
- m->num_fixed_variables = n_variables;
-}
-
-
-int module_get_fixed_var( struct module_t * m_, OBJECT * name )
-{
- struct fixed_variable * v;
- struct module_t * m = m_;
-
- if ( m->class_module )
- m = m->class_module;
-
- if ( !m->variable_indices )
- return -1;
-
- v = (struct fixed_variable *)hash_find( m->variable_indices, name );
- return v && v->n < m_->num_fixed_variables ? v->n : -1;
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/jam-files/engine/modules.h
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/jam-files/engine/modules.h b/ext/kenlm/jam-files/engine/modules.h
deleted file mode 100644
index 1b161c6..0000000
--- a/ext/kenlm/jam-files/engine/modules.h
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright 2001-2004 David Abrahams.
- * Distributed under the Boost Software License, Version 1.0.
- * (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
- */
-#ifndef MODULES_DWA10182001_H
-#define MODULES_DWA10182001_H
-
-#include "lists.h"
-#include "object.h"
-
-typedef struct module_t module_t ;
-struct module_t
-{
- OBJECT * name;
- struct hash * rules;
- struct hash * variables;
- struct hash * variable_indices;
- int num_fixed_variables;
- LIST * * fixed_variables;
- struct hash * imported_modules;
- module_t * class_module;
- struct hash * native_rules;
- int user_module;
-};
-
-module_t * bindmodule( OBJECT * name );
-module_t * root_module();
-void delete_module( module_t * );
-
-void import_module( LIST * module_names, module_t * target_module );
-LIST * imported_modules( module_t * );
-
-struct hash * demand_rules( module_t * );
-
-void module_bind_variables( module_t * );
-
-/*
- * After calling module_add_fixed_var, module_set_fixed_variables must be called
- * before accessing any variables in the module.
- */
-int module_add_fixed_var( module_t *, OBJECT * name, int * n );
-void module_set_fixed_variables( module_t *, int n );
-
-/*
- * Returns the index of the variable or -1 if none exists.
- */
-int module_get_fixed_var( module_t *, OBJECT * name );
-
-void modules_done();
-
-#endif