You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/04/19 21:33:54 UTC
[06/51] [partial] incubator-joshua git commit: Converted KenLM into a
submodule
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6da3961b/ext/kenlm/python/kenlm.cpp
----------------------------------------------------------------------
diff --git a/ext/kenlm b/ext/kenlm
new file mode 160000
index 0000000..56fdb5c
--- /dev/null
+++ b/ext/kenlm
@@ -0,0 +1 @@
+Subproject commit 56fdb5c44fca34d5a2e07d96139c28fb163983c5
diff --git a/ext/kenlm/python/kenlm.cpp b/ext/kenlm/python/kenlm.cpp
deleted file mode 100644
index 5bf47ff..0000000
--- a/ext/kenlm/python/kenlm.cpp
+++ /dev/null
@@ -1,7379 +0,0 @@
-/* Generated by Cython 0.22 */
-
-#define PY_SSIZE_T_CLEAN
-#ifndef CYTHON_USE_PYLONG_INTERNALS
-#ifdef PYLONG_BITS_IN_DIGIT
-#define CYTHON_USE_PYLONG_INTERNALS 0
-#else
-#include "pyconfig.h"
-#ifdef PYLONG_BITS_IN_DIGIT
-#define CYTHON_USE_PYLONG_INTERNALS 1
-#else
-#define CYTHON_USE_PYLONG_INTERNALS 0
-#endif
-#endif
-#endif
-#include "Python.h"
-#ifndef Py_PYTHON_H
- #error Python headers needed to compile C extensions, please install development version of Python.
-#elif PY_VERSION_HEX < 0x02060000 || (0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03020000)
- #error Cython requires Python 2.6+ or Python 3.2+.
-#else
-#define CYTHON_ABI "0_22"
-#include <stddef.h>
-#ifndef offsetof
-#define offsetof(type, member) ( (size_t) & ((type*)0) -> member )
-#endif
-#if !defined(WIN32) && !defined(MS_WINDOWS)
- #ifndef __stdcall
- #define __stdcall
- #endif
- #ifndef __cdecl
- #define __cdecl
- #endif
- #ifndef __fastcall
- #define __fastcall
- #endif
-#endif
-#ifndef DL_IMPORT
- #define DL_IMPORT(t) t
-#endif
-#ifndef DL_EXPORT
- #define DL_EXPORT(t) t
-#endif
-#ifndef PY_LONG_LONG
- #define PY_LONG_LONG LONG_LONG
-#endif
-#ifndef Py_HUGE_VAL
- #define Py_HUGE_VAL HUGE_VAL
-#endif
-#ifdef PYPY_VERSION
-#define CYTHON_COMPILING_IN_PYPY 1
-#define CYTHON_COMPILING_IN_CPYTHON 0
-#else
-#define CYTHON_COMPILING_IN_PYPY 0
-#define CYTHON_COMPILING_IN_CPYTHON 1
-#endif
-#if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x02070600 && !defined(Py_OptimizeFlag)
-#define Py_OptimizeFlag 0
-#endif
-#define __PYX_BUILD_PY_SSIZE_T "n"
-#define CYTHON_FORMAT_SSIZE_T "z"
-#if PY_MAJOR_VERSION < 3
- #define __Pyx_BUILTIN_MODULE_NAME "__builtin__"
- #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \
- PyCode_New(a+k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)
- #define __Pyx_DefaultClassType PyClass_Type
-#else
- #define __Pyx_BUILTIN_MODULE_NAME "builtins"
- #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \
- PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)
- #define __Pyx_DefaultClassType PyType_Type
-#endif
-#if PY_MAJOR_VERSION >= 3
- #define Py_TPFLAGS_CHECKTYPES 0
- #define Py_TPFLAGS_HAVE_INDEX 0
- #define Py_TPFLAGS_HAVE_NEWBUFFER 0
-#endif
-#if PY_VERSION_HEX < 0x030400a1 && !defined(Py_TPFLAGS_HAVE_FINALIZE)
- #define Py_TPFLAGS_HAVE_FINALIZE 0
-#endif
-#if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND)
- #define CYTHON_PEP393_ENABLED 1
- #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ? \
- 0 : _PyUnicode_Ready((PyObject *)(op)))
- #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_LENGTH(u)
- #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i)
- #define __Pyx_PyUnicode_KIND(u) PyUnicode_KIND(u)
- #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u)
- #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i)
-#else
- #define CYTHON_PEP393_ENABLED 0
- #define __Pyx_PyUnicode_READY(op) (0)
- #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_SIZE(u)
- #define __Pyx_PyUnicode_READ_CHAR(u, i) ((Py_UCS4)(PyUnicode_AS_UNICODE(u)[i]))
- #define __Pyx_PyUnicode_KIND(u) (sizeof(Py_UNICODE))
- #define __Pyx_PyUnicode_DATA(u) ((void*)PyUnicode_AS_UNICODE(u))
- #define __Pyx_PyUnicode_READ(k, d, i) ((void)(k), (Py_UCS4)(((Py_UNICODE*)d)[i]))
-#endif
-#if CYTHON_COMPILING_IN_PYPY
- #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b)
- #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b)
- #define __Pyx_PyFrozenSet_Size(s) PyObject_Size(s)
-#else
- #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b)
- #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ? \
- PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b))
- #define __Pyx_PyFrozenSet_Size(s) PySet_Size(s)
-#endif
-#define __Pyx_PyString_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : __Pyx_PyString_Format(a, b))
-#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b))
-#if PY_MAJOR_VERSION >= 3
- #define __Pyx_PyString_Format(a, b) PyUnicode_Format(a, b)
-#else
- #define __Pyx_PyString_Format(a, b) PyString_Format(a, b)
-#endif
-#if PY_MAJOR_VERSION >= 3
- #define PyBaseString_Type PyUnicode_Type
- #define PyStringObject PyUnicodeObject
- #define PyString_Type PyUnicode_Type
- #define PyString_Check PyUnicode_Check
- #define PyString_CheckExact PyUnicode_CheckExact
-#endif
-#if PY_MAJOR_VERSION >= 3
- #define __Pyx_PyBaseString_Check(obj) PyUnicode_Check(obj)
- #define __Pyx_PyBaseString_CheckExact(obj) PyUnicode_CheckExact(obj)
-#else
- #define __Pyx_PyBaseString_Check(obj) (PyString_Check(obj) || PyUnicode_Check(obj))
- #define __Pyx_PyBaseString_CheckExact(obj) (PyString_CheckExact(obj) || PyUnicode_CheckExact(obj))
-#endif
-#ifndef PySet_CheckExact
- #define PySet_CheckExact(obj) (Py_TYPE(obj) == &PySet_Type)
-#endif
-#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type)
-#if PY_MAJOR_VERSION >= 3
- #define PyIntObject PyLongObject
- #define PyInt_Type PyLong_Type
- #define PyInt_Check(op) PyLong_Check(op)
- #define PyInt_CheckExact(op) PyLong_CheckExact(op)
- #define PyInt_FromString PyLong_FromString
- #define PyInt_FromUnicode PyLong_FromUnicode
- #define PyInt_FromLong PyLong_FromLong
- #define PyInt_FromSize_t PyLong_FromSize_t
- #define PyInt_FromSsize_t PyLong_FromSsize_t
- #define PyInt_AsLong PyLong_AsLong
- #define PyInt_AS_LONG PyLong_AS_LONG
- #define PyInt_AsSsize_t PyLong_AsSsize_t
- #define PyInt_AsUnsignedLongMask PyLong_AsUnsignedLongMask
- #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask
- #define PyNumber_Int PyNumber_Long
-#endif
-#if PY_MAJOR_VERSION >= 3
- #define PyBoolObject PyLongObject
-#endif
-#if PY_MAJOR_VERSION >= 3 && CYTHON_COMPILING_IN_PYPY
- #ifndef PyUnicode_InternFromString
- #define PyUnicode_InternFromString(s) PyUnicode_FromString(s)
- #endif
-#endif
-#if PY_VERSION_HEX < 0x030200A4
- typedef long Py_hash_t;
- #define __Pyx_PyInt_FromHash_t PyInt_FromLong
- #define __Pyx_PyInt_AsHash_t PyInt_AsLong
-#else
- #define __Pyx_PyInt_FromHash_t PyInt_FromSsize_t
- #define __Pyx_PyInt_AsHash_t PyInt_AsSsize_t
-#endif
-#if PY_MAJOR_VERSION >= 3
- #define __Pyx_PyMethod_New(func, self, klass) ((self) ? PyMethod_New(func, self) : PyInstanceMethod_New(func))
-#else
- #define __Pyx_PyMethod_New(func, self, klass) PyMethod_New(func, self, klass)
-#endif
-#ifndef CYTHON_INLINE
- #if defined(__GNUC__)
- #define CYTHON_INLINE __inline__
- #elif defined(_MSC_VER)
- #define CYTHON_INLINE __inline
- #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
- #define CYTHON_INLINE inline
- #else
- #define CYTHON_INLINE
- #endif
-#endif
-#ifndef CYTHON_RESTRICT
- #if defined(__GNUC__)
- #define CYTHON_RESTRICT __restrict__
- #elif defined(_MSC_VER) && _MSC_VER >= 1400
- #define CYTHON_RESTRICT __restrict
- #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
- #define CYTHON_RESTRICT restrict
- #else
- #define CYTHON_RESTRICT
- #endif
-#endif
-#ifdef NAN
-#define __PYX_NAN() ((float) NAN)
-#else
-static CYTHON_INLINE float __PYX_NAN() {
- /* Initialize NaN. The sign is irrelevant, an exponent with all bits 1 and
- a nonzero mantissa means NaN. If the first bit in the mantissa is 1, it is
- a quiet NaN. */
- float value;
- memset(&value, 0xFF, sizeof(value));
- return value;
-}
-#endif
-#define __Pyx_void_to_None(void_result) (void_result, Py_INCREF(Py_None), Py_None)
-#ifdef __cplusplus
-template<typename T>
-void __Pyx_call_destructor(T* x) {
- x->~T();
-}
-template<typename T>
-class __Pyx_FakeReference {
- public:
- __Pyx_FakeReference() : ptr(NULL) { }
- __Pyx_FakeReference(T& ref) : ptr(&ref) { }
- T *operator->() { return ptr; }
- operator T&() { return *ptr; }
- private:
- T *ptr;
-};
-#endif
-
-
-#if PY_MAJOR_VERSION >= 3
- #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y)
- #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y)
-#else
- #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y)
- #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceDivide(x,y)
-#endif
-
-#ifndef __PYX_EXTERN_C
- #ifdef __cplusplus
- #define __PYX_EXTERN_C extern "C"
- #else
- #define __PYX_EXTERN_C extern
- #endif
-#endif
-
-#if defined(WIN32) || defined(MS_WINDOWS)
-#define _USE_MATH_DEFINES
-#endif
-#include <math.h>
-#define __PYX_HAVE__kenlm
-#define __PYX_HAVE_API__kenlm
-#include "lm/word_index.hh"
-#include "lm/return.hh"
-#include "lm/state.hh"
-#include "lm/virtual_interface.hh"
-#include "lm/model.hh"
-#include "ios"
-#include "new"
-#include "stdexcept"
-#include "typeinfo"
-#ifdef _OPENMP
-#include <omp.h>
-#endif /* _OPENMP */
-
-#ifdef PYREX_WITHOUT_ASSERTIONS
-#define CYTHON_WITHOUT_ASSERTIONS
-#endif
-
-#ifndef CYTHON_UNUSED
-# if defined(__GNUC__)
-# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4))
-# define CYTHON_UNUSED __attribute__ ((__unused__))
-# else
-# define CYTHON_UNUSED
-# endif
-# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER))
-# define CYTHON_UNUSED __attribute__ ((__unused__))
-# else
-# define CYTHON_UNUSED
-# endif
-#endif
-typedef struct {PyObject **p; char *s; const Py_ssize_t n; const char* encoding;
- const char is_unicode; const char is_str; const char intern; } __Pyx_StringTabEntry;
-
-#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0
-#define __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT 0
-#define __PYX_DEFAULT_STRING_ENCODING ""
-#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString
-#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize
-#define __Pyx_fits_Py_ssize_t(v, type, is_signed) ( \
- (sizeof(type) < sizeof(Py_ssize_t)) || \
- (sizeof(type) > sizeof(Py_ssize_t) && \
- likely(v < (type)PY_SSIZE_T_MAX || \
- v == (type)PY_SSIZE_T_MAX) && \
- (!is_signed || likely(v > (type)PY_SSIZE_T_MIN || \
- v == (type)PY_SSIZE_T_MIN))) || \
- (sizeof(type) == sizeof(Py_ssize_t) && \
- (is_signed || likely(v < (type)PY_SSIZE_T_MAX || \
- v == (type)PY_SSIZE_T_MAX))) )
-static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject*);
-static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length);
-#define __Pyx_PyByteArray_FromString(s) PyByteArray_FromStringAndSize((const char*)s, strlen((const char*)s))
-#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l)
-#define __Pyx_PyBytes_FromString PyBytes_FromString
-#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize
-static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*);
-#if PY_MAJOR_VERSION < 3
- #define __Pyx_PyStr_FromString __Pyx_PyBytes_FromString
- #define __Pyx_PyStr_FromStringAndSize __Pyx_PyBytes_FromStringAndSize
-#else
- #define __Pyx_PyStr_FromString __Pyx_PyUnicode_FromString
- #define __Pyx_PyStr_FromStringAndSize __Pyx_PyUnicode_FromStringAndSize
-#endif
-#define __Pyx_PyObject_AsSString(s) ((signed char*) __Pyx_PyObject_AsString(s))
-#define __Pyx_PyObject_AsUString(s) ((unsigned char*) __Pyx_PyObject_AsString(s))
-#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s)
-#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s)
-#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s)
-#define __Pyx_PyStr_FromCString(s) __Pyx_PyStr_FromString((const char*)s)
-#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s)
-#if PY_MAJOR_VERSION < 3
-static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u)
-{
- const Py_UNICODE *u_end = u;
- while (*u_end++) ;
- return (size_t)(u_end - u - 1);
-}
-#else
-#define __Pyx_Py_UNICODE_strlen Py_UNICODE_strlen
-#endif
-#define __Pyx_PyUnicode_FromUnicode(u) PyUnicode_FromUnicode(u, __Pyx_Py_UNICODE_strlen(u))
-#define __Pyx_PyUnicode_FromUnicodeAndLength PyUnicode_FromUnicode
-#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode
-#define __Pyx_Owned_Py_None(b) (Py_INCREF(Py_None), Py_None)
-#define __Pyx_PyBool_FromLong(b) ((b) ? (Py_INCREF(Py_True), Py_True) : (Py_INCREF(Py_False), Py_False))
-static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*);
-static CYTHON_INLINE PyObject* __Pyx_PyNumber_Int(PyObject* x);
-static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*);
-static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t);
-#if CYTHON_COMPILING_IN_CPYTHON
-#define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x))
-#else
-#define __pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x)
-#endif
-#define __pyx_PyFloat_AsFloat(x) ((float) __pyx_PyFloat_AsDouble(x))
-#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII
-static int __Pyx_sys_getdefaultencoding_not_ascii;
-static int __Pyx_init_sys_getdefaultencoding_params(void) {
- PyObject* sys;
- PyObject* default_encoding = NULL;
- PyObject* ascii_chars_u = NULL;
- PyObject* ascii_chars_b = NULL;
- const char* default_encoding_c;
- sys = PyImport_ImportModule("sys");
- if (!sys) goto bad;
- default_encoding = PyObject_CallMethod(sys, (char*) "getdefaultencoding", NULL);
- Py_DECREF(sys);
- if (!default_encoding) goto bad;
- default_encoding_c = PyBytes_AsString(default_encoding);
- if (!default_encoding_c) goto bad;
- if (strcmp(default_encoding_c, "ascii") == 0) {
- __Pyx_sys_getdefaultencoding_not_ascii = 0;
- } else {
- char ascii_chars[128];
- int c;
- for (c = 0; c < 128; c++) {
- ascii_chars[c] = c;
- }
- __Pyx_sys_getdefaultencoding_not_ascii = 1;
- ascii_chars_u = PyUnicode_DecodeASCII(ascii_chars, 128, NULL);
- if (!ascii_chars_u) goto bad;
- ascii_chars_b = PyUnicode_AsEncodedString(ascii_chars_u, default_encoding_c, NULL);
- if (!ascii_chars_b || !PyBytes_Check(ascii_chars_b) || memcmp(ascii_chars, PyBytes_AS_STRING(ascii_chars_b), 128) != 0) {
- PyErr_Format(
- PyExc_ValueError,
- "This module compiled with c_string_encoding=ascii, but default encoding '%.200s' is not a superset of ascii.",
- default_encoding_c);
- goto bad;
- }
- Py_DECREF(ascii_chars_u);
- Py_DECREF(ascii_chars_b);
- }
- Py_DECREF(default_encoding);
- return 0;
-bad:
- Py_XDECREF(default_encoding);
- Py_XDECREF(ascii_chars_u);
- Py_XDECREF(ascii_chars_b);
- return -1;
-}
-#endif
-#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT && PY_MAJOR_VERSION >= 3
-#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL)
-#else
-#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL)
-#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT
-static char* __PYX_DEFAULT_STRING_ENCODING;
-static int __Pyx_init_sys_getdefaultencoding_params(void) {
- PyObject* sys;
- PyObject* default_encoding = NULL;
- char* default_encoding_c;
- sys = PyImport_ImportModule("sys");
- if (!sys) goto bad;
- default_encoding = PyObject_CallMethod(sys, (char*) (const char*) "getdefaultencoding", NULL);
- Py_DECREF(sys);
- if (!default_encoding) goto bad;
- default_encoding_c = PyBytes_AsString(default_encoding);
- if (!default_encoding_c) goto bad;
- __PYX_DEFAULT_STRING_ENCODING = (char*) malloc(strlen(default_encoding_c));
- if (!__PYX_DEFAULT_STRING_ENCODING) goto bad;
- strcpy(__PYX_DEFAULT_STRING_ENCODING, default_encoding_c);
- Py_DECREF(default_encoding);
- return 0;
-bad:
- Py_XDECREF(default_encoding);
- return -1;
-}
-#endif
-#endif
-
-
-/* Test for GCC > 2.95 */
-#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95)))
- #define likely(x) __builtin_expect(!!(x), 1)
- #define unlikely(x) __builtin_expect(!!(x), 0)
-#else /* !__GNUC__ or GCC < 2.95 */
- #define likely(x) (x)
- #define unlikely(x) (x)
-#endif /* __GNUC__ */
-
-static PyObject *__pyx_m;
-static PyObject *__pyx_d;
-static PyObject *__pyx_b;
-static PyObject *__pyx_empty_tuple;
-static PyObject *__pyx_empty_bytes;
-static int __pyx_lineno;
-static int __pyx_clineno = 0;
-static const char * __pyx_cfilenm= __FILE__;
-static const char *__pyx_filename;
-
-
-static const char *__pyx_f[] = {
- "kenlm.pyx",
-};
-
-/*--- Type declarations ---*/
-struct __pyx_obj_5kenlm_FullScoreReturn;
-struct __pyx_obj_5kenlm_State;
-struct __pyx_obj_5kenlm_LanguageModel;
-struct __pyx_obj_5kenlm_Model;
-struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores;
-
-/* "kenlm.pyx":11
- * raise TypeError('Cannot convert %s to string' % type(data))
- *
- * cdef class FullScoreReturn: # <<<<<<<<<<<<<<
- * """
- * Wrapper around FullScoreReturn.
- */
-struct __pyx_obj_5kenlm_FullScoreReturn {
- PyObject_HEAD
- float log_prob;
- int ngram_length;
- int oov;
-};
-
-
-/* "kenlm.pyx":44
- * return self.oov
- *
- * cdef class State: # <<<<<<<<<<<<<<
- * """
- * Wrapper around lm::ngram::State so that python code can make incremental queries.
- */
-struct __pyx_obj_5kenlm_State {
- PyObject_HEAD
- lm::ngram::State _c_state;
-};
-
-
-/* "kenlm.pyx":74
- *
- *
- * cdef class LanguageModel: # <<<<<<<<<<<<<<
- * """
- * This is not a strict wrapper, the interface is more pythonic.
- */
-struct __pyx_obj_5kenlm_LanguageModel {
- PyObject_HEAD
- lm::base::Model *model;
- PyObject *path;
- const lm::base::Vocabulary *vocab;
-};
-
-
-/* "kenlm.pyx":159
- * return (_kenlm.LanguageModel, (self.path,))
- *
- * cdef class Model: # <<<<<<<<<<<<<<
- * """
- * This is closer to a wrapper around lm::ngram::Model.
- */
-struct __pyx_obj_5kenlm_Model {
- PyObject_HEAD
- lm::base::Model *model;
- PyObject *path;
- const lm::base::Vocabulary *vocab;
-};
-
-
-/* "kenlm.pyx":122
- * return total
- *
- * def full_scores(self, sentence, bos = True, eos = True): # <<<<<<<<<<<<<<
- * """
- * full_scores(sentence, bos = True, eos = Ture) -> generate full scores (prob, ngram length, oov)
- */
-struct __pyx_obj_5kenlm___pyx_scope_struct__full_scores {
- PyObject_HEAD
- PyObject *__pyx_v_bos;
- PyObject *__pyx_v_eos;
- lm::ngram::State __pyx_v_out_state;
- struct lm::FullScoreReturn __pyx_v_ret;
- struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self;
- PyObject *__pyx_v_sentence;
- lm::ngram::State __pyx_v_state;
- float __pyx_v_total;
- lm::WordIndex __pyx_v_wid;
- PyObject *__pyx_v_word;
- PyObject *__pyx_v_words;
- PyObject *__pyx_t_0;
- Py_ssize_t __pyx_t_1;
-};
-
-
-/* --- Runtime support code (head) --- */
-#ifndef CYTHON_REFNANNY
- #define CYTHON_REFNANNY 0
-#endif
-#if CYTHON_REFNANNY
- typedef struct {
- void (*INCREF)(void*, PyObject*, int);
- void (*DECREF)(void*, PyObject*, int);
- void (*GOTREF)(void*, PyObject*, int);
- void (*GIVEREF)(void*, PyObject*, int);
- void* (*SetupContext)(const char*, int, const char*);
- void (*FinishContext)(void**);
- } __Pyx_RefNannyAPIStruct;
- static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL;
- static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname);
- #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL;
-#ifdef WITH_THREAD
- #define __Pyx_RefNannySetupContext(name, acquire_gil) \
- if (acquire_gil) { \
- PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure(); \
- __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__); \
- PyGILState_Release(__pyx_gilstate_save); \
- } else { \
- __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__); \
- }
-#else
- #define __Pyx_RefNannySetupContext(name, acquire_gil) \
- __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__)
-#endif
- #define __Pyx_RefNannyFinishContext() \
- __Pyx_RefNanny->FinishContext(&__pyx_refnanny)
- #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), __LINE__)
- #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), __LINE__)
- #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), __LINE__)
- #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), __LINE__)
- #define __Pyx_XINCREF(r) do { if((r) != NULL) {__Pyx_INCREF(r); }} while(0)
- #define __Pyx_XDECREF(r) do { if((r) != NULL) {__Pyx_DECREF(r); }} while(0)
- #define __Pyx_XGOTREF(r) do { if((r) != NULL) {__Pyx_GOTREF(r); }} while(0)
- #define __Pyx_XGIVEREF(r) do { if((r) != NULL) {__Pyx_GIVEREF(r);}} while(0)
-#else
- #define __Pyx_RefNannyDeclarations
- #define __Pyx_RefNannySetupContext(name, acquire_gil)
- #define __Pyx_RefNannyFinishContext()
- #define __Pyx_INCREF(r) Py_INCREF(r)
- #define __Pyx_DECREF(r) Py_DECREF(r)
- #define __Pyx_GOTREF(r)
- #define __Pyx_GIVEREF(r)
- #define __Pyx_XINCREF(r) Py_XINCREF(r)
- #define __Pyx_XDECREF(r) Py_XDECREF(r)
- #define __Pyx_XGOTREF(r)
- #define __Pyx_XGIVEREF(r)
-#endif
-#define __Pyx_XDECREF_SET(r, v) do { \
- PyObject *tmp = (PyObject *) r; \
- r = v; __Pyx_XDECREF(tmp); \
- } while (0)
-#define __Pyx_DECREF_SET(r, v) do { \
- PyObject *tmp = (PyObject *) r; \
- r = v; __Pyx_DECREF(tmp); \
- } while (0)
-#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0)
-#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0)
-
-#if CYTHON_COMPILING_IN_CPYTHON
-static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) {
- PyTypeObject* tp = Py_TYPE(obj);
- if (likely(tp->tp_getattro))
- return tp->tp_getattro(obj, attr_name);
-#if PY_MAJOR_VERSION < 3
- if (likely(tp->tp_getattr))
- return tp->tp_getattr(obj, PyString_AS_STRING(attr_name));
-#endif
- return PyObject_GetAttr(obj, attr_name);
-}
-#else
-#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n)
-#endif
-
-static PyObject *__Pyx_GetBuiltinName(PyObject *name);
-
-#if CYTHON_COMPILING_IN_CPYTHON
-static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw);
-#else
-#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw)
-#endif
-
-static CYTHON_INLINE void __Pyx_ErrRestore(PyObject *type, PyObject *value, PyObject *tb);
-static CYTHON_INLINE void __Pyx_ErrFetch(PyObject **type, PyObject **value, PyObject **tb);
-
-static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause);
-
-static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact,
- Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found);
-
-static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name);
-
-static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[], \
- PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args, \
- const char* function_name);
-
-static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed,
- const char *name, int exact);
-
-static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name);
-
-#if CYTHON_COMPILING_IN_CPYTHON
-static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg);
-#endif
-
-static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg);
-
-static CYTHON_INLINE void __Pyx_ExceptionSave(PyObject **type, PyObject **value, PyObject **tb);
-static void __Pyx_ExceptionReset(PyObject *type, PyObject *value, PyObject *tb);
-
-static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb);
-
-#if CYTHON_COMPILING_IN_CPYTHON
-static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func);
-#else
-#define __Pyx_PyObject_CallNoArg(func) __Pyx_PyObject_Call(func, __pyx_empty_tuple, NULL)
-#endif
-
-#include <string.h>
-
-typedef struct {
- int code_line;
- PyCodeObject* code_object;
-} __Pyx_CodeObjectCacheEntry;
-struct __Pyx_CodeObjectCache {
- int count;
- int max_count;
- __Pyx_CodeObjectCacheEntry* entries;
-};
-static struct __Pyx_CodeObjectCache __pyx_code_cache = {0,0,NULL};
-static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line);
-static PyCodeObject *__pyx_find_code_object(int code_line);
-static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object);
-
-static void __Pyx_AddTraceback(const char *funcname, int c_line,
- int py_line, const char *filename);
-
-#include <new>
-
-static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level);
-
-static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *);
-
-static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value);
-
-#ifndef __Pyx_CppExn2PyErr
-#include <new>
-#include <typeinfo>
-#include <stdexcept>
-#include <ios>
-static void __Pyx_CppExn2PyErr() {
- try {
- if (PyErr_Occurred())
- ; // let the latest Python exn pass through and ignore the current one
- else
- throw;
- } catch (const std::bad_alloc& exn) {
- PyErr_SetString(PyExc_MemoryError, exn.what());
- } catch (const std::bad_cast& exn) {
- PyErr_SetString(PyExc_TypeError, exn.what());
- } catch (const std::domain_error& exn) {
- PyErr_SetString(PyExc_ValueError, exn.what());
- } catch (const std::invalid_argument& exn) {
- PyErr_SetString(PyExc_ValueError, exn.what());
- } catch (const std::ios_base::failure& exn) {
- PyErr_SetString(PyExc_IOError, exn.what());
- } catch (const std::out_of_range& exn) {
- PyErr_SetString(PyExc_IndexError, exn.what());
- } catch (const std::overflow_error& exn) {
- PyErr_SetString(PyExc_OverflowError, exn.what());
- } catch (const std::range_error& exn) {
- PyErr_SetString(PyExc_ArithmeticError, exn.what());
- } catch (const std::underflow_error& exn) {
- PyErr_SetString(PyExc_ArithmeticError, exn.what());
- } catch (const std::exception& exn) {
- PyErr_SetString(PyExc_RuntimeError, exn.what());
- }
- catch (...)
- {
- PyErr_SetString(PyExc_RuntimeError, "Unknown exception");
- }
-}
-#endif
-
-static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_int(unsigned int value);
-
-static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_char(unsigned char value);
-
-static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value);
-
-static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *);
-
-static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type);
-
-static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb);
-
-static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg);
-
-#define __Pyx_Generator_USED
-#include <structmember.h>
-#include <frameobject.h>
-typedef PyObject *(*__pyx_generator_body_t)(PyObject *, PyObject *);
-typedef struct {
- PyObject_HEAD
- __pyx_generator_body_t body;
- PyObject *closure;
- PyObject *exc_type;
- PyObject *exc_value;
- PyObject *exc_traceback;
- PyObject *gi_weakreflist;
- PyObject *classobj;
- PyObject *yieldfrom;
- PyObject *gi_name;
- PyObject *gi_qualname;
- int resume_label;
- char is_running;
-} __pyx_GeneratorObject;
-static __pyx_GeneratorObject *__Pyx_Generator_New(__pyx_generator_body_t body,
- PyObject *closure, PyObject *name, PyObject *qualname);
-static int __pyx_Generator_init(void);
-static int __Pyx_Generator_clear(PyObject* self);
-#if 1 || PY_VERSION_HEX < 0x030300B0
-static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue);
-#else
-#define __Pyx_PyGen_FetchStopIterationValue(pvalue) PyGen_FetchStopIterationValue(pvalue)
-#endif
-
-static int __Pyx_check_binary_version(void);
-
-static int __Pyx_InitStrings(__Pyx_StringTabEntry *t);
-
-
-/* Module declarations from '_kenlm' */
-
-/* Module declarations from 'kenlm' */
-static PyTypeObject *__pyx_ptype_5kenlm_FullScoreReturn = 0;
-static PyTypeObject *__pyx_ptype_5kenlm_State = 0;
-static PyTypeObject *__pyx_ptype_5kenlm_LanguageModel = 0;
-static PyTypeObject *__pyx_ptype_5kenlm_Model = 0;
-static PyTypeObject *__pyx_ptype_5kenlm___pyx_scope_struct__full_scores = 0;
-static PyObject *__pyx_f_5kenlm_as_str(PyObject *); /*proto*/
-#define __Pyx_MODULE_NAME "kenlm"
-int __pyx_module_is_main_kenlm = 0;
-
-/* Implementation of 'kenlm' */
-static PyObject *__pyx_builtin_TypeError;
-static PyObject *__pyx_builtin_RuntimeError;
-static PyObject *__pyx_builtin_IOError;
-static int __pyx_pf_5kenlm_15FullScoreReturn___cinit__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self, PyObject *__pyx_v_log_prob, PyObject *__pyx_v_ngram_length, PyObject *__pyx_v_oov); /* proto */
-static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_2__repr__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self); /* proto */
-static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_8log_prob___get__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self); /* proto */
-static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_12ngram_length___get__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self); /* proto */
-static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_3oov___get__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self); /* proto */
-static PyObject *__pyx_pf_5kenlm_5State___richcmp__(struct __pyx_obj_5kenlm_State *__pyx_v_qa, struct __pyx_obj_5kenlm_State *__pyx_v_qb, int __pyx_v_op); /* proto */
-static Py_hash_t __pyx_pf_5kenlm_5State_2__hash__(struct __pyx_obj_5kenlm_State *__pyx_v_self); /* proto */
-static int __pyx_pf_5kenlm_13LanguageModel___init__(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self, PyObject *__pyx_v_path); /* proto */
-static void __pyx_pf_5kenlm_13LanguageModel_2__dealloc__(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self); /* proto */
-static PyObject *__pyx_pf_5kenlm_13LanguageModel_5order___get__(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self); /* proto */
-static PyObject *__pyx_pf_5kenlm_13LanguageModel_4score(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self, PyObject *__pyx_v_sentence, PyObject *__pyx_v_bos, PyObject *__pyx_v_eos); /* proto */
-static PyObject *__pyx_pf_5kenlm_13LanguageModel_6full_scores(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self, PyObject *__pyx_v_sentence, PyObject *__pyx_v_bos, PyObject *__pyx_v_eos); /* proto */
-static int __pyx_pf_5kenlm_13LanguageModel_9__contains__(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self, PyObject *__pyx_v_word); /* proto */
-static PyObject *__pyx_pf_5kenlm_13LanguageModel_11__repr__(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self); /* proto */
-static PyObject *__pyx_pf_5kenlm_13LanguageModel_13__reduce__(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self); /* proto */
-static PyObject *__pyx_pf_5kenlm_13LanguageModel_4path___get__(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self); /* proto */
-static int __pyx_pf_5kenlm_13LanguageModel_4path_2__set__(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self, PyObject *__pyx_v_value); /* proto */
-static int __pyx_pf_5kenlm_13LanguageModel_4path_4__del__(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self); /* proto */
-static int __pyx_pf_5kenlm_5Model___init__(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_path); /* proto */
-static void __pyx_pf_5kenlm_5Model_2__dealloc__(struct __pyx_obj_5kenlm_Model *__pyx_v_self); /* proto */
-static PyObject *__pyx_pf_5kenlm_5Model_5order___get__(struct __pyx_obj_5kenlm_Model *__pyx_v_self); /* proto */
-static PyObject *__pyx_pf_5kenlm_5Model_4BeginSentenceWrite(struct __pyx_obj_5kenlm_Model *__pyx_v_self, struct __pyx_obj_5kenlm_State *__pyx_v_state); /* proto */
-static PyObject *__pyx_pf_5kenlm_5Model_6NullContextWrite(struct __pyx_obj_5kenlm_Model *__pyx_v_self, struct __pyx_obj_5kenlm_State *__pyx_v_state); /* proto */
-static PyObject *__pyx_pf_5kenlm_5Model_8BaseScore(struct __pyx_obj_5kenlm_Model *__pyx_v_self, struct __pyx_obj_5kenlm_State *__pyx_v_in_state, PyObject *__pyx_v_word, struct __pyx_obj_5kenlm_State *__pyx_v_out_state); /* proto */
-static PyObject *__pyx_pf_5kenlm_5Model_10BaseFullScore(struct __pyx_obj_5kenlm_Model *__pyx_v_self, struct __pyx_obj_5kenlm_State *__pyx_v_in_state, PyObject *__pyx_v_word, struct __pyx_obj_5kenlm_State *__pyx_v_out_state); /* proto */
-static int __pyx_pf_5kenlm_5Model_12__contains__(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_word); /* proto */
-static PyObject *__pyx_pf_5kenlm_5Model_14__repr__(struct __pyx_obj_5kenlm_Model *__pyx_v_self); /* proto */
-static PyObject *__pyx_pf_5kenlm_5Model_16__reduce__(struct __pyx_obj_5kenlm_Model *__pyx_v_self); /* proto */
-static PyObject *__pyx_pf_5kenlm_5Model_4path___get__(struct __pyx_obj_5kenlm_Model *__pyx_v_self); /* proto */
-static int __pyx_pf_5kenlm_5Model_4path_2__set__(struct __pyx_obj_5kenlm_Model *__pyx_v_self, PyObject *__pyx_v_value); /* proto */
-static int __pyx_pf_5kenlm_5Model_4path_4__del__(struct __pyx_obj_5kenlm_Model *__pyx_v_self); /* proto */
-static PyObject *__pyx_tp_new_5kenlm_FullScoreReturn(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/
-static PyObject *__pyx_tp_new_5kenlm_State(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/
-static PyObject *__pyx_tp_new_5kenlm_LanguageModel(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/
-static PyObject *__pyx_tp_new_5kenlm_Model(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/
-static PyObject *__pyx_tp_new_5kenlm___pyx_scope_struct__full_scores(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/
-static char __pyx_k__2[] = "\n";
-static char __pyx_k__3[] = " ";
-static char __pyx_k_os[] = "os";
-static char __pyx_k_bos[] = "bos";
-static char __pyx_k_eos[] = "eos";
-static char __pyx_k_oov[] = "oov";
-static char __pyx_k_args[] = "args";
-static char __pyx_k_main[] = "__main__";
-static char __pyx_k_name[] = "__name__";
-static char __pyx_k_path[] = "path";
-static char __pyx_k_send[] = "send";
-static char __pyx_k_test[] = "__test__";
-static char __pyx_k_utf8[] = "utf8";
-static char __pyx_k_word[] = "word";
-static char __pyx_k_class[] = "__class__";
-static char __pyx_k_close[] = "close";
-static char __pyx_k_kenlm[] = "_kenlm";
-static char __pyx_k_split[] = "split";
-static char __pyx_k_throw[] = "throw";
-static char __pyx_k_encode[] = "encode";
-static char __pyx_k_format[] = "format";
-static char __pyx_k_import[] = "__import__";
-static char __pyx_k_0_1_2_3[] = "{0}({1}, {2}, {3})";
-static char __pyx_k_IOError[] = "IOError";
-static char __pyx_k_abspath[] = "abspath";
-static char __pyx_k_replace[] = "replace";
-static char __pyx_k_basename[] = "basename";
-static char __pyx_k_in_state[] = "in_state";
-static char __pyx_k_log_prob[] = "log_prob";
-static char __pyx_k_sentence[] = "sentence";
-static char __pyx_k_TypeError[] = "TypeError";
-static char __pyx_k_out_state[] = "out_state";
-static char __pyx_k_full_scores[] = "full_scores";
-static char __pyx_k_Model_from_0[] = "<Model from {0}>";
-static char __pyx_k_RuntimeError[] = "RuntimeError";
-static char __pyx_k_ngram_length[] = "ngram_length";
-static char __pyx_k_LanguageModel[] = "LanguageModel";
-static char __pyx_k_Cannot_read_model[] = "Cannot read model '{}' ({})";
-static char __pyx_k_LanguageModel_from_0[] = "<LanguageModel from {0}>";
-static char __pyx_k_LanguageModel_full_scores[] = "LanguageModel.full_scores";
-static char __pyx_k_Cannot_convert_s_to_string[] = "Cannot convert %s to string";
-static PyObject *__pyx_kp_s_0_1_2_3;
-static PyObject *__pyx_kp_s_Cannot_convert_s_to_string;
-static PyObject *__pyx_kp_s_Cannot_read_model;
-static PyObject *__pyx_n_s_IOError;
-static PyObject *__pyx_n_s_LanguageModel;
-static PyObject *__pyx_kp_s_LanguageModel_from_0;
-static PyObject *__pyx_n_s_LanguageModel_full_scores;
-static PyObject *__pyx_kp_s_Model_from_0;
-static PyObject *__pyx_n_s_RuntimeError;
-static PyObject *__pyx_n_s_TypeError;
-static PyObject *__pyx_kp_s__2;
-static PyObject *__pyx_kp_s__3;
-static PyObject *__pyx_n_s_abspath;
-static PyObject *__pyx_n_s_args;
-static PyObject *__pyx_n_s_basename;
-static PyObject *__pyx_n_s_bos;
-static PyObject *__pyx_n_s_class;
-static PyObject *__pyx_n_s_close;
-static PyObject *__pyx_n_s_encode;
-static PyObject *__pyx_n_s_eos;
-static PyObject *__pyx_n_s_format;
-static PyObject *__pyx_n_s_full_scores;
-static PyObject *__pyx_n_s_import;
-static PyObject *__pyx_n_s_in_state;
-static PyObject *__pyx_n_s_kenlm;
-static PyObject *__pyx_n_s_log_prob;
-static PyObject *__pyx_n_s_main;
-static PyObject *__pyx_n_s_name;
-static PyObject *__pyx_n_s_ngram_length;
-static PyObject *__pyx_n_s_oov;
-static PyObject *__pyx_n_s_os;
-static PyObject *__pyx_n_s_out_state;
-static PyObject *__pyx_n_s_path;
-static PyObject *__pyx_n_s_replace;
-static PyObject *__pyx_n_s_send;
-static PyObject *__pyx_n_s_sentence;
-static PyObject *__pyx_n_s_split;
-static PyObject *__pyx_n_s_test;
-static PyObject *__pyx_n_s_throw;
-static PyObject *__pyx_n_s_utf8;
-static PyObject *__pyx_n_s_word;
-static PyObject *__pyx_tuple_;
-static PyObject *__pyx_tuple__4;
-static PyObject *__pyx_tuple__5;
-
-/* "kenlm.pyx":4
- * cimport _kenlm
- *
- * cdef bytes as_str(data): # <<<<<<<<<<<<<<
- * if isinstance(data, bytes):
- * return data
- */
-
-static PyObject *__pyx_f_5kenlm_as_str(PyObject *__pyx_v_data) {
- PyObject *__pyx_r = NULL;
- __Pyx_RefNannyDeclarations
- int __pyx_t_1;
- int __pyx_t_2;
- PyObject *__pyx_t_3 = NULL;
- PyObject *__pyx_t_4 = NULL;
- int __pyx_lineno = 0;
- const char *__pyx_filename = NULL;
- int __pyx_clineno = 0;
- __Pyx_RefNannySetupContext("as_str", 0);
-
- /* "kenlm.pyx":5
- *
- * cdef bytes as_str(data):
- * if isinstance(data, bytes): # <<<<<<<<<<<<<<
- * return data
- * elif isinstance(data, unicode):
- */
- __pyx_t_1 = PyBytes_Check(__pyx_v_data);
- __pyx_t_2 = (__pyx_t_1 != 0);
- if (__pyx_t_2) {
-
- /* "kenlm.pyx":6
- * cdef bytes as_str(data):
- * if isinstance(data, bytes):
- * return data # <<<<<<<<<<<<<<
- * elif isinstance(data, unicode):
- * return data.encode('utf8')
- */
- __Pyx_XDECREF(__pyx_r);
- if (!(likely(PyBytes_CheckExact(__pyx_v_data))||((__pyx_v_data) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_v_data)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 6; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_INCREF(__pyx_v_data);
- __pyx_r = ((PyObject*)__pyx_v_data);
- goto __pyx_L0;
- }
-
- /* "kenlm.pyx":7
- * if isinstance(data, bytes):
- * return data
- * elif isinstance(data, unicode): # <<<<<<<<<<<<<<
- * return data.encode('utf8')
- * raise TypeError('Cannot convert %s to string' % type(data))
- */
- __pyx_t_2 = PyUnicode_Check(__pyx_v_data);
- __pyx_t_1 = (__pyx_t_2 != 0);
- if (__pyx_t_1) {
-
- /* "kenlm.pyx":8
- * return data
- * elif isinstance(data, unicode):
- * return data.encode('utf8') # <<<<<<<<<<<<<<
- * raise TypeError('Cannot convert %s to string' % type(data))
- *
- */
- __Pyx_XDECREF(__pyx_r);
- __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_data, __pyx_n_s_encode); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 8; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_3);
- __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 8; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_4);
- __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
- if (!(likely(PyBytes_CheckExact(__pyx_t_4))||((__pyx_t_4) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_t_4)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 8; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __pyx_r = ((PyObject*)__pyx_t_4);
- __pyx_t_4 = 0;
- goto __pyx_L0;
- }
-
- /* "kenlm.pyx":9
- * elif isinstance(data, unicode):
- * return data.encode('utf8')
- * raise TypeError('Cannot convert %s to string' % type(data)) # <<<<<<<<<<<<<<
- *
- * cdef class FullScoreReturn:
- */
- __pyx_t_4 = __Pyx_PyString_Format(__pyx_kp_s_Cannot_convert_s_to_string, ((PyObject *)Py_TYPE(__pyx_v_data))); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 9; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_4);
- __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 9; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_3);
- PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_4);
- __Pyx_GIVEREF(__pyx_t_4);
- __pyx_t_4 = 0;
- __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_t_3, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 9; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_4);
- __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
- __Pyx_Raise(__pyx_t_4, 0, 0, 0);
- __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
- {__pyx_filename = __pyx_f[0]; __pyx_lineno = 9; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-
- /* "kenlm.pyx":4
- * cimport _kenlm
- *
- * cdef bytes as_str(data): # <<<<<<<<<<<<<<
- * if isinstance(data, bytes):
- * return data
- */
-
- /* function exit code */
- __pyx_L1_error:;
- __Pyx_XDECREF(__pyx_t_3);
- __Pyx_XDECREF(__pyx_t_4);
- __Pyx_AddTraceback("kenlm.as_str", __pyx_clineno, __pyx_lineno, __pyx_filename);
- __pyx_r = 0;
- __pyx_L0:;
- __Pyx_XGIVEREF(__pyx_r);
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-/* "kenlm.pyx":24
- * cdef bint oov
- *
- * def __cinit__(self, log_prob, ngram_length, oov): # <<<<<<<<<<<<<<
- * self.log_prob = log_prob
- * self.ngram_length = ngram_length
- */
-
-/* Python wrapper */
-static int __pyx_pw_5kenlm_15FullScoreReturn_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
-static int __pyx_pw_5kenlm_15FullScoreReturn_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
- PyObject *__pyx_v_log_prob = 0;
- PyObject *__pyx_v_ngram_length = 0;
- PyObject *__pyx_v_oov = 0;
- int __pyx_lineno = 0;
- const char *__pyx_filename = NULL;
- int __pyx_clineno = 0;
- int __pyx_r;
- __Pyx_RefNannyDeclarations
- __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0);
- {
- static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_log_prob,&__pyx_n_s_ngram_length,&__pyx_n_s_oov,0};
- PyObject* values[3] = {0,0,0};
- if (unlikely(__pyx_kwds)) {
- Py_ssize_t kw_args;
- const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args);
- switch (pos_args) {
- case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2);
- case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1);
- case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
- case 0: break;
- default: goto __pyx_L5_argtuple_error;
- }
- kw_args = PyDict_Size(__pyx_kwds);
- switch (pos_args) {
- case 0:
- if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_log_prob)) != 0)) kw_args--;
- else goto __pyx_L5_argtuple_error;
- case 1:
- if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_ngram_length)) != 0)) kw_args--;
- else {
- __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 3, 3, 1); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 24; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
- }
- case 2:
- if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_oov)) != 0)) kw_args--;
- else {
- __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 3, 3, 2); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 24; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
- }
- }
- if (unlikely(kw_args > 0)) {
- if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 24; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
- }
- } else if (PyTuple_GET_SIZE(__pyx_args) != 3) {
- goto __pyx_L5_argtuple_error;
- } else {
- values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
- values[1] = PyTuple_GET_ITEM(__pyx_args, 1);
- values[2] = PyTuple_GET_ITEM(__pyx_args, 2);
- }
- __pyx_v_log_prob = values[0];
- __pyx_v_ngram_length = values[1];
- __pyx_v_oov = values[2];
- }
- goto __pyx_L4_argument_unpacking_done;
- __pyx_L5_argtuple_error:;
- __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 24; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
- __pyx_L3_error:;
- __Pyx_AddTraceback("kenlm.FullScoreReturn.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename);
- __Pyx_RefNannyFinishContext();
- return -1;
- __pyx_L4_argument_unpacking_done:;
- __pyx_r = __pyx_pf_5kenlm_15FullScoreReturn___cinit__(((struct __pyx_obj_5kenlm_FullScoreReturn *)__pyx_v_self), __pyx_v_log_prob, __pyx_v_ngram_length, __pyx_v_oov);
-
- /* function exit code */
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-static int __pyx_pf_5kenlm_15FullScoreReturn___cinit__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self, PyObject *__pyx_v_log_prob, PyObject *__pyx_v_ngram_length, PyObject *__pyx_v_oov) {
- int __pyx_r;
- __Pyx_RefNannyDeclarations
- float __pyx_t_1;
- int __pyx_t_2;
- int __pyx_t_3;
- int __pyx_lineno = 0;
- const char *__pyx_filename = NULL;
- int __pyx_clineno = 0;
- __Pyx_RefNannySetupContext("__cinit__", 0);
-
- /* "kenlm.pyx":25
- *
- * def __cinit__(self, log_prob, ngram_length, oov):
- * self.log_prob = log_prob # <<<<<<<<<<<<<<
- * self.ngram_length = ngram_length
- * self.oov = oov
- */
- __pyx_t_1 = __pyx_PyFloat_AsFloat(__pyx_v_log_prob); if (unlikely((__pyx_t_1 == (float)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 25; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __pyx_v_self->log_prob = __pyx_t_1;
-
- /* "kenlm.pyx":26
- * def __cinit__(self, log_prob, ngram_length, oov):
- * self.log_prob = log_prob
- * self.ngram_length = ngram_length # <<<<<<<<<<<<<<
- * self.oov = oov
- *
- */
- __pyx_t_2 = __Pyx_PyInt_As_int(__pyx_v_ngram_length); if (unlikely((__pyx_t_2 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 26; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __pyx_v_self->ngram_length = __pyx_t_2;
-
- /* "kenlm.pyx":27
- * self.log_prob = log_prob
- * self.ngram_length = ngram_length
- * self.oov = oov # <<<<<<<<<<<<<<
- *
- * def __repr__(self):
- */
- __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_v_oov); if (unlikely((__pyx_t_3 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 27; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __pyx_v_self->oov = __pyx_t_3;
-
- /* "kenlm.pyx":24
- * cdef bint oov
- *
- * def __cinit__(self, log_prob, ngram_length, oov): # <<<<<<<<<<<<<<
- * self.log_prob = log_prob
- * self.ngram_length = ngram_length
- */
-
- /* function exit code */
- __pyx_r = 0;
- goto __pyx_L0;
- __pyx_L1_error:;
- __Pyx_AddTraceback("kenlm.FullScoreReturn.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename);
- __pyx_r = -1;
- __pyx_L0:;
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-/* "kenlm.pyx":29
- * self.oov = oov
- *
- * def __repr__(self): # <<<<<<<<<<<<<<
- * return '{0}({1}, {2}, {3})'.format(self.__class__.__name__, repr(self.log_prob), repr(self.ngram_length), repr(self.oov))
- *
- */
-
-/* Python wrapper */
-static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_3__repr__(PyObject *__pyx_v_self); /*proto*/
-static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_3__repr__(PyObject *__pyx_v_self) {
- PyObject *__pyx_r = 0;
- __Pyx_RefNannyDeclarations
- __Pyx_RefNannySetupContext("__repr__ (wrapper)", 0);
- __pyx_r = __pyx_pf_5kenlm_15FullScoreReturn_2__repr__(((struct __pyx_obj_5kenlm_FullScoreReturn *)__pyx_v_self));
-
- /* function exit code */
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_2__repr__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self) {
- PyObject *__pyx_r = NULL;
- __Pyx_RefNannyDeclarations
- PyObject *__pyx_t_1 = NULL;
- PyObject *__pyx_t_2 = NULL;
- PyObject *__pyx_t_3 = NULL;
- PyObject *__pyx_t_4 = NULL;
- PyObject *__pyx_t_5 = NULL;
- PyObject *__pyx_t_6 = NULL;
- PyObject *__pyx_t_7 = NULL;
- Py_ssize_t __pyx_t_8;
- PyObject *__pyx_t_9 = NULL;
- int __pyx_lineno = 0;
- const char *__pyx_filename = NULL;
- int __pyx_clineno = 0;
- __Pyx_RefNannySetupContext("__repr__", 0);
-
- /* "kenlm.pyx":30
- *
- * def __repr__(self):
- * return '{0}({1}, {2}, {3})'.format(self.__class__.__name__, repr(self.log_prob), repr(self.ngram_length), repr(self.oov)) # <<<<<<<<<<<<<<
- *
- * property log_prob:
- */
- __Pyx_XDECREF(__pyx_r);
- __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_kp_s_0_1_2_3, __pyx_n_s_format); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_2);
- __pyx_t_3 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_n_s_class); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_3);
- __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_name); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_4);
- __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
- __pyx_t_3 = PyFloat_FromDouble(__pyx_v_self->log_prob); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_3);
- __pyx_t_5 = PyObject_Repr(__pyx_t_3); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_5);
- __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
- __pyx_t_3 = __Pyx_PyInt_From_int(__pyx_v_self->ngram_length); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_3);
- __pyx_t_6 = PyObject_Repr(__pyx_t_3); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_6);
- __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
- __pyx_t_3 = __Pyx_PyBool_FromLong(__pyx_v_self->oov); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_3);
- __pyx_t_7 = PyObject_Repr(__pyx_t_3); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_7);
- __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
- __pyx_t_3 = NULL;
- __pyx_t_8 = 0;
- if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_2))) {
- __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_2);
- if (likely(__pyx_t_3)) {
- PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2);
- __Pyx_INCREF(__pyx_t_3);
- __Pyx_INCREF(function);
- __Pyx_DECREF_SET(__pyx_t_2, function);
- __pyx_t_8 = 1;
- }
- }
- __pyx_t_9 = PyTuple_New(4+__pyx_t_8); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_9);
- if (__pyx_t_3) {
- PyTuple_SET_ITEM(__pyx_t_9, 0, __pyx_t_3); __Pyx_GIVEREF(__pyx_t_3); __pyx_t_3 = NULL;
- }
- PyTuple_SET_ITEM(__pyx_t_9, 0+__pyx_t_8, __pyx_t_4);
- __Pyx_GIVEREF(__pyx_t_4);
- PyTuple_SET_ITEM(__pyx_t_9, 1+__pyx_t_8, __pyx_t_5);
- __Pyx_GIVEREF(__pyx_t_5);
- PyTuple_SET_ITEM(__pyx_t_9, 2+__pyx_t_8, __pyx_t_6);
- __Pyx_GIVEREF(__pyx_t_6);
- PyTuple_SET_ITEM(__pyx_t_9, 3+__pyx_t_8, __pyx_t_7);
- __Pyx_GIVEREF(__pyx_t_7);
- __pyx_t_4 = 0;
- __pyx_t_5 = 0;
- __pyx_t_6 = 0;
- __pyx_t_7 = 0;
- __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_9, NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 30; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_1);
- __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
- __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
- __pyx_r = __pyx_t_1;
- __pyx_t_1 = 0;
- goto __pyx_L0;
-
- /* "kenlm.pyx":29
- * self.oov = oov
- *
- * def __repr__(self): # <<<<<<<<<<<<<<
- * return '{0}({1}, {2}, {3})'.format(self.__class__.__name__, repr(self.log_prob), repr(self.ngram_length), repr(self.oov))
- *
- */
-
- /* function exit code */
- __pyx_L1_error:;
- __Pyx_XDECREF(__pyx_t_1);
- __Pyx_XDECREF(__pyx_t_2);
- __Pyx_XDECREF(__pyx_t_3);
- __Pyx_XDECREF(__pyx_t_4);
- __Pyx_XDECREF(__pyx_t_5);
- __Pyx_XDECREF(__pyx_t_6);
- __Pyx_XDECREF(__pyx_t_7);
- __Pyx_XDECREF(__pyx_t_9);
- __Pyx_AddTraceback("kenlm.FullScoreReturn.__repr__", __pyx_clineno, __pyx_lineno, __pyx_filename);
- __pyx_r = NULL;
- __pyx_L0:;
- __Pyx_XGIVEREF(__pyx_r);
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-/* "kenlm.pyx":33
- *
- * property log_prob:
- * def __get__(self): # <<<<<<<<<<<<<<
- * return self.log_prob
- *
- */
-
-/* Python wrapper */
-static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_8log_prob_1__get__(PyObject *__pyx_v_self); /*proto*/
-static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_8log_prob_1__get__(PyObject *__pyx_v_self) {
- PyObject *__pyx_r = 0;
- __Pyx_RefNannyDeclarations
- __Pyx_RefNannySetupContext("__get__ (wrapper)", 0);
- __pyx_r = __pyx_pf_5kenlm_15FullScoreReturn_8log_prob___get__(((struct __pyx_obj_5kenlm_FullScoreReturn *)__pyx_v_self));
-
- /* function exit code */
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_8log_prob___get__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self) {
- PyObject *__pyx_r = NULL;
- __Pyx_RefNannyDeclarations
- PyObject *__pyx_t_1 = NULL;
- int __pyx_lineno = 0;
- const char *__pyx_filename = NULL;
- int __pyx_clineno = 0;
- __Pyx_RefNannySetupContext("__get__", 0);
-
- /* "kenlm.pyx":34
- * property log_prob:
- * def __get__(self):
- * return self.log_prob # <<<<<<<<<<<<<<
- *
- * property ngram_length:
- */
- __Pyx_XDECREF(__pyx_r);
- __pyx_t_1 = PyFloat_FromDouble(__pyx_v_self->log_prob); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 34; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_1);
- __pyx_r = __pyx_t_1;
- __pyx_t_1 = 0;
- goto __pyx_L0;
-
- /* "kenlm.pyx":33
- *
- * property log_prob:
- * def __get__(self): # <<<<<<<<<<<<<<
- * return self.log_prob
- *
- */
-
- /* function exit code */
- __pyx_L1_error:;
- __Pyx_XDECREF(__pyx_t_1);
- __Pyx_AddTraceback("kenlm.FullScoreReturn.log_prob.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename);
- __pyx_r = NULL;
- __pyx_L0:;
- __Pyx_XGIVEREF(__pyx_r);
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-/* "kenlm.pyx":37
- *
- * property ngram_length:
- * def __get__(self): # <<<<<<<<<<<<<<
- * return self.ngram_length
- *
- */
-
-/* Python wrapper */
-static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_12ngram_length_1__get__(PyObject *__pyx_v_self); /*proto*/
-static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_12ngram_length_1__get__(PyObject *__pyx_v_self) {
- PyObject *__pyx_r = 0;
- __Pyx_RefNannyDeclarations
- __Pyx_RefNannySetupContext("__get__ (wrapper)", 0);
- __pyx_r = __pyx_pf_5kenlm_15FullScoreReturn_12ngram_length___get__(((struct __pyx_obj_5kenlm_FullScoreReturn *)__pyx_v_self));
-
- /* function exit code */
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_12ngram_length___get__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self) {
- PyObject *__pyx_r = NULL;
- __Pyx_RefNannyDeclarations
- PyObject *__pyx_t_1 = NULL;
- int __pyx_lineno = 0;
- const char *__pyx_filename = NULL;
- int __pyx_clineno = 0;
- __Pyx_RefNannySetupContext("__get__", 0);
-
- /* "kenlm.pyx":38
- * property ngram_length:
- * def __get__(self):
- * return self.ngram_length # <<<<<<<<<<<<<<
- *
- * property oov:
- */
- __Pyx_XDECREF(__pyx_r);
- __pyx_t_1 = __Pyx_PyInt_From_int(__pyx_v_self->ngram_length); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 38; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_1);
- __pyx_r = __pyx_t_1;
- __pyx_t_1 = 0;
- goto __pyx_L0;
-
- /* "kenlm.pyx":37
- *
- * property ngram_length:
- * def __get__(self): # <<<<<<<<<<<<<<
- * return self.ngram_length
- *
- */
-
- /* function exit code */
- __pyx_L1_error:;
- __Pyx_XDECREF(__pyx_t_1);
- __Pyx_AddTraceback("kenlm.FullScoreReturn.ngram_length.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename);
- __pyx_r = NULL;
- __pyx_L0:;
- __Pyx_XGIVEREF(__pyx_r);
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-/* "kenlm.pyx":41
- *
- * property oov:
- * def __get__(self): # <<<<<<<<<<<<<<
- * return self.oov
- *
- */
-
-/* Python wrapper */
-static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_3oov_1__get__(PyObject *__pyx_v_self); /*proto*/
-static PyObject *__pyx_pw_5kenlm_15FullScoreReturn_3oov_1__get__(PyObject *__pyx_v_self) {
- PyObject *__pyx_r = 0;
- __Pyx_RefNannyDeclarations
- __Pyx_RefNannySetupContext("__get__ (wrapper)", 0);
- __pyx_r = __pyx_pf_5kenlm_15FullScoreReturn_3oov___get__(((struct __pyx_obj_5kenlm_FullScoreReturn *)__pyx_v_self));
-
- /* function exit code */
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-static PyObject *__pyx_pf_5kenlm_15FullScoreReturn_3oov___get__(struct __pyx_obj_5kenlm_FullScoreReturn *__pyx_v_self) {
- PyObject *__pyx_r = NULL;
- __Pyx_RefNannyDeclarations
- PyObject *__pyx_t_1 = NULL;
- int __pyx_lineno = 0;
- const char *__pyx_filename = NULL;
- int __pyx_clineno = 0;
- __Pyx_RefNannySetupContext("__get__", 0);
-
- /* "kenlm.pyx":42
- * property oov:
- * def __get__(self):
- * return self.oov # <<<<<<<<<<<<<<
- *
- * cdef class State:
- */
- __Pyx_XDECREF(__pyx_r);
- __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->oov); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 42; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_1);
- __pyx_r = __pyx_t_1;
- __pyx_t_1 = 0;
- goto __pyx_L0;
-
- /* "kenlm.pyx":41
- *
- * property oov:
- * def __get__(self): # <<<<<<<<<<<<<<
- * return self.oov
- *
- */
-
- /* function exit code */
- __pyx_L1_error:;
- __Pyx_XDECREF(__pyx_t_1);
- __Pyx_AddTraceback("kenlm.FullScoreReturn.oov.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename);
- __pyx_r = NULL;
- __pyx_L0:;
- __Pyx_XGIVEREF(__pyx_r);
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-/* "kenlm.pyx":55
- * cdef _kenlm.State _c_state
- *
- * def __richcmp__(State qa, State qb, int op): # <<<<<<<<<<<<<<
- * r = qa._c_state.Compare(qb._c_state)
- * if op == 0: # <
- */
-
-/* Python wrapper */
-static PyObject *__pyx_pw_5kenlm_5State_1__richcmp__(PyObject *__pyx_v_qa, PyObject *__pyx_v_qb, int __pyx_v_op); /*proto*/
-static PyObject *__pyx_pw_5kenlm_5State_1__richcmp__(PyObject *__pyx_v_qa, PyObject *__pyx_v_qb, int __pyx_v_op) {
- CYTHON_UNUSED int __pyx_lineno = 0;
- CYTHON_UNUSED const char *__pyx_filename = NULL;
- CYTHON_UNUSED int __pyx_clineno = 0;
- PyObject *__pyx_r = 0;
- __Pyx_RefNannyDeclarations
- __Pyx_RefNannySetupContext("__richcmp__ (wrapper)", 0);
- if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_qa), __pyx_ptype_5kenlm_State, 1, "qa", 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 55; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_qb), __pyx_ptype_5kenlm_State, 1, "qb", 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 55; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __pyx_r = __pyx_pf_5kenlm_5State___richcmp__(((struct __pyx_obj_5kenlm_State *)__pyx_v_qa), ((struct __pyx_obj_5kenlm_State *)__pyx_v_qb), ((int)__pyx_v_op));
-
- /* function exit code */
- goto __pyx_L0;
- __pyx_L1_error:;
- __pyx_r = NULL;
- __pyx_L0:;
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-static PyObject *__pyx_pf_5kenlm_5State___richcmp__(struct __pyx_obj_5kenlm_State *__pyx_v_qa, struct __pyx_obj_5kenlm_State *__pyx_v_qb, int __pyx_v_op) {
- int __pyx_v_r;
- PyObject *__pyx_r = NULL;
- __Pyx_RefNannyDeclarations
- PyObject *__pyx_t_1 = NULL;
- int __pyx_lineno = 0;
- const char *__pyx_filename = NULL;
- int __pyx_clineno = 0;
- __Pyx_RefNannySetupContext("__richcmp__", 0);
-
- /* "kenlm.pyx":56
- *
- * def __richcmp__(State qa, State qb, int op):
- * r = qa._c_state.Compare(qb._c_state) # <<<<<<<<<<<<<<
- * if op == 0: # <
- * return r < 0
- */
- __pyx_v_r = __pyx_v_qa->_c_state.Compare(__pyx_v_qb->_c_state);
-
- /* "kenlm.pyx":65
- * elif op == 3: # !=
- * return r != 0
- * elif op == 4: # > # <<<<<<<<<<<<<<
- * return r > 0
- * else: # >=
- */
- switch (__pyx_v_op) {
-
- /* "kenlm.pyx":57
- * def __richcmp__(State qa, State qb, int op):
- * r = qa._c_state.Compare(qb._c_state)
- * if op == 0: # < # <<<<<<<<<<<<<<
- * return r < 0
- * elif op == 1: # <=
- */
- case 0:
-
- /* "kenlm.pyx":58
- * r = qa._c_state.Compare(qb._c_state)
- * if op == 0: # <
- * return r < 0 # <<<<<<<<<<<<<<
- * elif op == 1: # <=
- * return r <= 0
- */
- __Pyx_XDECREF(__pyx_r);
- __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_r < 0)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 58; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_1);
- __pyx_r = __pyx_t_1;
- __pyx_t_1 = 0;
- goto __pyx_L0;
- break;
-
- /* "kenlm.pyx":59
- * if op == 0: # <
- * return r < 0
- * elif op == 1: # <= # <<<<<<<<<<<<<<
- * return r <= 0
- * elif op == 2: # ==
- */
- case 1:
-
- /* "kenlm.pyx":60
- * return r < 0
- * elif op == 1: # <=
- * return r <= 0 # <<<<<<<<<<<<<<
- * elif op == 2: # ==
- * return r == 0
- */
- __Pyx_XDECREF(__pyx_r);
- __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_r <= 0)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 60; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_1);
- __pyx_r = __pyx_t_1;
- __pyx_t_1 = 0;
- goto __pyx_L0;
- break;
-
- /* "kenlm.pyx":61
- * elif op == 1: # <=
- * return r <= 0
- * elif op == 2: # == # <<<<<<<<<<<<<<
- * return r == 0
- * elif op == 3: # !=
- */
- case 2:
-
- /* "kenlm.pyx":62
- * return r <= 0
- * elif op == 2: # ==
- * return r == 0 # <<<<<<<<<<<<<<
- * elif op == 3: # !=
- * return r != 0
- */
- __Pyx_XDECREF(__pyx_r);
- __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_r == 0)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_1);
- __pyx_r = __pyx_t_1;
- __pyx_t_1 = 0;
- goto __pyx_L0;
- break;
-
- /* "kenlm.pyx":63
- * elif op == 2: # ==
- * return r == 0
- * elif op == 3: # != # <<<<<<<<<<<<<<
- * return r != 0
- * elif op == 4: # >
- */
- case 3:
-
- /* "kenlm.pyx":64
- * return r == 0
- * elif op == 3: # !=
- * return r != 0 # <<<<<<<<<<<<<<
- * elif op == 4: # >
- * return r > 0
- */
- __Pyx_XDECREF(__pyx_r);
- __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_r != 0)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 64; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_1);
- __pyx_r = __pyx_t_1;
- __pyx_t_1 = 0;
- goto __pyx_L0;
- break;
-
- /* "kenlm.pyx":65
- * elif op == 3: # !=
- * return r != 0
- * elif op == 4: # > # <<<<<<<<<<<<<<
- * return r > 0
- * else: # >=
- */
- case 4:
-
- /* "kenlm.pyx":66
- * return r != 0
- * elif op == 4: # >
- * return r > 0 # <<<<<<<<<<<<<<
- * else: # >=
- * return r >= 0
- */
- __Pyx_XDECREF(__pyx_r);
- __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_r > 0)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_1);
- __pyx_r = __pyx_t_1;
- __pyx_t_1 = 0;
- goto __pyx_L0;
- break;
- default:
-
- /* "kenlm.pyx":68
- * return r > 0
- * else: # >=
- * return r >= 0 # <<<<<<<<<<<<<<
- *
- * def __hash__(self):
- */
- __Pyx_XDECREF(__pyx_r);
- __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_r >= 0)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_1);
- __pyx_r = __pyx_t_1;
- __pyx_t_1 = 0;
- goto __pyx_L0;
- break;
- }
-
- /* "kenlm.pyx":55
- * cdef _kenlm.State _c_state
- *
- * def __richcmp__(State qa, State qb, int op): # <<<<<<<<<<<<<<
- * r = qa._c_state.Compare(qb._c_state)
- * if op == 0: # <
- */
-
- /* function exit code */
- __pyx_L1_error:;
- __Pyx_XDECREF(__pyx_t_1);
- __Pyx_AddTraceback("kenlm.State.__richcmp__", __pyx_clineno, __pyx_lineno, __pyx_filename);
- __pyx_r = NULL;
- __pyx_L0:;
- __Pyx_XGIVEREF(__pyx_r);
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-/* "kenlm.pyx":70
- * return r >= 0
- *
- * def __hash__(self): # <<<<<<<<<<<<<<
- * return _kenlm.hash_value(self._c_state)
- *
- */
-
-/* Python wrapper */
-static Py_hash_t __pyx_pw_5kenlm_5State_3__hash__(PyObject *__pyx_v_self); /*proto*/
-static Py_hash_t __pyx_pw_5kenlm_5State_3__hash__(PyObject *__pyx_v_self) {
- Py_hash_t __pyx_r;
- __Pyx_RefNannyDeclarations
- __Pyx_RefNannySetupContext("__hash__ (wrapper)", 0);
- __pyx_r = __pyx_pf_5kenlm_5State_2__hash__(((struct __pyx_obj_5kenlm_State *)__pyx_v_self));
-
- /* function exit code */
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-static Py_hash_t __pyx_pf_5kenlm_5State_2__hash__(struct __pyx_obj_5kenlm_State *__pyx_v_self) {
- Py_hash_t __pyx_r;
- __Pyx_RefNannyDeclarations
- __Pyx_RefNannySetupContext("__hash__", 0);
-
- /* "kenlm.pyx":71
- *
- * def __hash__(self):
- * return _kenlm.hash_value(self._c_state) # <<<<<<<<<<<<<<
- *
- *
- */
- __pyx_r = lm::ngram::hash_value(__pyx_v_self->_c_state);
- goto __pyx_L0;
-
- /* "kenlm.pyx":70
- * return r >= 0
- *
- * def __hash__(self): # <<<<<<<<<<<<<<
- * return _kenlm.hash_value(self._c_state)
- *
- */
-
- /* function exit code */
- __pyx_L0:;
- if (unlikely(__pyx_r == -1) && !PyErr_Occurred()) __pyx_r = -2;
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-/* "kenlm.pyx":84
- * cdef _kenlm.const_Vocabulary* vocab
- *
- * def __init__(self, path): # <<<<<<<<<<<<<<
- * """
- * Load the language model.
- */
-
-/* Python wrapper */
-static int __pyx_pw_5kenlm_13LanguageModel_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
-static char __pyx_doc_5kenlm_13LanguageModel___init__[] = "\n Load the language model.\n\n :param path: path to an arpa file or a kenlm binary file.\n ";
-#if CYTHON_COMPILING_IN_CPYTHON
-struct wrapperbase __pyx_wrapperbase_5kenlm_13LanguageModel___init__;
-#endif
-static int __pyx_pw_5kenlm_13LanguageModel_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
- PyObject *__pyx_v_path = 0;
- int __pyx_lineno = 0;
- const char *__pyx_filename = NULL;
- int __pyx_clineno = 0;
- int __pyx_r;
- __Pyx_RefNannyDeclarations
- __Pyx_RefNannySetupContext("__init__ (wrapper)", 0);
- {
- static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_path,0};
- PyObject* values[1] = {0};
- if (unlikely(__pyx_kwds)) {
- Py_ssize_t kw_args;
- const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args);
- switch (pos_args) {
- case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
- case 0: break;
- default: goto __pyx_L5_argtuple_error;
- }
- kw_args = PyDict_Size(__pyx_kwds);
- switch (pos_args) {
- case 0:
- if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_path)) != 0)) kw_args--;
- else goto __pyx_L5_argtuple_error;
- }
- if (unlikely(kw_args > 0)) {
- if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
- }
- } else if (PyTuple_GET_SIZE(__pyx_args) != 1) {
- goto __pyx_L5_argtuple_error;
- } else {
- values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
- }
- __pyx_v_path = values[0];
- }
- goto __pyx_L4_argument_unpacking_done;
- __pyx_L5_argtuple_error:;
- __Pyx_RaiseArgtupleInvalid("__init__", 1, 1, 1, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
- __pyx_L3_error:;
- __Pyx_AddTraceback("kenlm.LanguageModel.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename);
- __Pyx_RefNannyFinishContext();
- return -1;
- __pyx_L4_argument_unpacking_done:;
- __pyx_r = __pyx_pf_5kenlm_13LanguageModel___init__(((struct __pyx_obj_5kenlm_LanguageModel *)__pyx_v_self), __pyx_v_path);
-
- /* function exit code */
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-static int __pyx_pf_5kenlm_13LanguageModel___init__(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self, PyObject *__pyx_v_path) {
- PyObject *__pyx_v_exception = NULL;
- PyObject *__pyx_v_exception_message = NULL;
- int __pyx_r;
- __Pyx_RefNannyDeclarations
- PyObject *__pyx_t_1 = NULL;
- PyObject *__pyx_t_2 = NULL;
- PyObject *__pyx_t_3 = NULL;
- PyObject *__pyx_t_4 = NULL;
- PyObject *__pyx_t_5 = NULL;
- PyObject *__pyx_t_6 = NULL;
- PyObject *__pyx_t_7 = NULL;
- PyObject *__pyx_t_8 = NULL;
- char *__pyx_t_9;
- lm::base::Model *__pyx_t_10;
- int __pyx_t_11;
- PyObject *__pyx_t_12 = NULL;
- Py_ssize_t __pyx_t_13;
- PyObject *__pyx_t_14 = NULL;
- int __pyx_lineno = 0;
- const char *__pyx_filename = NULL;
- int __pyx_clineno = 0;
- __Pyx_RefNannySetupContext("__init__", 0);
-
- /* "kenlm.pyx":90
- * :param path: path to an arpa file or a kenlm binary file.
- * """
- * self.path = os.path.abspath(as_str(path)) # <<<<<<<<<<<<<<
- * try:
- * self.model = _kenlm.LoadVirtual(self.path)
- */
- __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_os); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_2);
- __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_path); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_3);
- __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
- __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_abspath); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_2);
- __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
- __pyx_t_3 = __pyx_f_5kenlm_as_str(__pyx_v_path); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_3);
- __pyx_t_4 = NULL;
- if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_2))) {
- __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_2);
- if (likely(__pyx_t_4)) {
- PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2);
- __Pyx_INCREF(__pyx_t_4);
- __Pyx_INCREF(function);
- __Pyx_DECREF_SET(__pyx_t_2, function);
- }
- }
- if (!__pyx_t_4) {
- __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
- __Pyx_GOTREF(__pyx_t_1);
- } else {
- __pyx_t_5 = PyTuple_New(1+1); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_5);
- PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_4); __Pyx_GIVEREF(__pyx_t_4); __pyx_t_4 = NULL;
- PyTuple_SET_ITEM(__pyx_t_5, 0+1, __pyx_t_3);
- __Pyx_GIVEREF(__pyx_t_3);
- __pyx_t_3 = 0;
- __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_1);
- __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
- }
- __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
- if (!(likely(PyBytes_CheckExact(__pyx_t_1))||((__pyx_t_1) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_t_1)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GIVEREF(__pyx_t_1);
- __Pyx_GOTREF(__pyx_v_self->path);
- __Pyx_DECREF(__pyx_v_self->path);
- __pyx_v_self->path = ((PyObject*)__pyx_t_1);
- __pyx_t_1 = 0;
-
- /* "kenlm.pyx":91
- * """
- * self.path = os.path.abspath(as_str(path))
- * try: # <<<<<<<<<<<<<<
- * self.model = _kenlm.LoadVirtual(self.path)
- * except RuntimeError as exception:
- */
- {
- __Pyx_ExceptionSave(&__pyx_t_6, &__pyx_t_7, &__pyx_t_8);
- __Pyx_XGOTREF(__pyx_t_6);
- __Pyx_XGOTREF(__pyx_t_7);
- __Pyx_XGOTREF(__pyx_t_8);
- /*try:*/ {
-
- /* "kenlm.pyx":92
- * self.path = os.path.abspath(as_str(path))
- * try:
- * self.model = _kenlm.LoadVirtual(self.path) # <<<<<<<<<<<<<<
- * except RuntimeError as exception:
- * exception_message = str(exception).replace('\n', ' ')
- */
- __pyx_t_9 = __Pyx_PyObject_AsString(__pyx_v_self->path); if (unlikely((!__pyx_t_9) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 92; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
- try {
- __pyx_t_10 = lm::ngram::LoadVirtual(__pyx_t_9);
- } catch(...) {
- __Pyx_CppExn2PyErr();
- {__pyx_filename = __pyx_f[0]; __pyx_lineno = 92; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
- }
- __pyx_v_self->model = __pyx_t_10;
- }
- __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0;
- __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0;
- __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0;
- goto __pyx_L10_try_end;
- __pyx_L3_error:;
- __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0;
- __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0;
- __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0;
- __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0;
- __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0;
-
- /* "kenlm.pyx":93
- * try:
- * self.model = _kenlm.LoadVirtual(self.path)
- * except RuntimeError as exception: # <<<<<<<<<<<<<<
- * exception_message = str(exception).replace('\n', ' ')
- * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\
- */
- __pyx_t_11 = PyErr_ExceptionMatches(__pyx_builtin_RuntimeError);
- if (__pyx_t_11) {
- __Pyx_AddTraceback("kenlm.LanguageModel.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename);
- if (__Pyx_GetException(&__pyx_t_1, &__pyx_t_2, &__pyx_t_5) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 93; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;}
- __Pyx_GOTREF(__pyx_t_1);
- __Pyx_GOTREF(__pyx_t_2);
- __Pyx_GOTREF(__pyx_t_5);
- __Pyx_INCREF(__pyx_t_2);
- __pyx_v_exception = __pyx_t_2;
-
- /* "kenlm.pyx":94
- * self.model = _kenlm.LoadVirtual(self.path)
- * except RuntimeError as exception:
- * exception_message = str(exception).replace('\n', ' ') # <<<<<<<<<<<<<<
- * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\
- * from exception
- */
- __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 94; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;}
- __Pyx_GOTREF(__pyx_t_3);
- __Pyx_INCREF(__pyx_v_exception);
- PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_v_exception);
- __Pyx_GIVEREF(__pyx_v_exception);
- __pyx_t_4 = __Pyx_PyObject_Call(((PyObject *)((PyObject*)(&PyString_Type))), __pyx_t_3, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 94; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;}
- __Pyx_GOTREF(__pyx_t_4);
- __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
- __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_replace); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 94; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;}
- __Pyx_GOTREF(__pyx_t_3);
- __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
- __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_tuple__4, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 94; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;}
- __Pyx_GOTREF(__pyx_t_4);
- __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
- __pyx_v_exception_message = __pyx_t_4;
- __pyx_t_4 = 0;
-
- /* "kenlm.pyx":95
- * except RuntimeError as exception:
- * exception_message = str(exception).replace('\n', ' ')
- * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\ # <<<<<<<<<<<<<<
- * from exception
- * self.vocab = &self.model.BaseVocabulary()
- */
- __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_kp_s_Cannot_read_model, __pyx_n_s_format); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;}
- __Pyx_GOTREF(__pyx_t_3);
- __pyx_t_12 = NULL;
- __pyx_t_13 = 0;
- if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_3))) {
- __pyx_t_12 = PyMethod_GET_SELF(__pyx_t_3);
- if (likely(__pyx_t_12)) {
- PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3);
- __Pyx_INCREF(__pyx_t_12);
- __Pyx_INCREF(function);
- __Pyx_DECREF_SET(__pyx_t_3, function);
- __pyx_t_13 = 1;
- }
- }
- __pyx_t_14 = PyTuple_New(2+__pyx_t_13); if (unlikely(!__pyx_t_14)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;}
- __Pyx_GOTREF(__pyx_t_14);
- if (__pyx_t_12) {
- PyTuple_SET_ITEM(__pyx_t_14, 0, __pyx_t_12); __Pyx_GIVEREF(__pyx_t_12); __pyx_t_12 = NULL;
- }
- __Pyx_INCREF(__pyx_v_path);
- PyTuple_SET_ITEM(__pyx_t_14, 0+__pyx_t_13, __pyx_v_path);
- __Pyx_GIVEREF(__pyx_v_path);
- __Pyx_INCREF(__pyx_v_exception_message);
- PyTuple_SET_ITEM(__pyx_t_14, 1+__pyx_t_13, __pyx_v_exception_message);
- __Pyx_GIVEREF(__pyx_v_exception_message);
- __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_14, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;}
- __Pyx_GOTREF(__pyx_t_4);
- __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0;
- __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
- __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;}
- __Pyx_GOTREF(__pyx_t_3);
- PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_4);
- __Pyx_GIVEREF(__pyx_t_4);
- __pyx_t_4 = 0;
- __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_IOError, __pyx_t_3, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;}
- __Pyx_GOTREF(__pyx_t_4);
- __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
-
- /* "kenlm.pyx":96
- * exception_message = str(exception).replace('\n', ' ')
- * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\
- * from exception # <<<<<<<<<<<<<<
- * self.vocab = &self.model.BaseVocabulary()
- *
- */
- __Pyx_Raise(__pyx_t_4, 0, 0, __pyx_v_exception);
- __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
- {__pyx_filename = __pyx_f[0]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L5_except_error;}
- }
- goto __pyx_L5_except_error;
- __pyx_L5_except_error:;
- __Pyx_XGIVEREF(__pyx_t_6);
- __Pyx_XGIVEREF(__pyx_t_7);
- __Pyx_XGIVEREF(__pyx_t_8);
- __Pyx_ExceptionReset(__pyx_t_6, __pyx_t_7, __pyx_t_8);
- goto __pyx_L1_error;
- __pyx_L10_try_end:;
- }
-
- /* "kenlm.pyx":97
- * raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\
- * from exception
- * self.vocab = &self.model.BaseVocabulary() # <<<<<<<<<<<<<<
- *
- * def __dealloc__(self):
- */
- __pyx_v_self->vocab = (&__pyx_v_self->model->BaseVocabulary());
-
- /* "kenlm.pyx":84
- * cdef _kenlm.const_Vocabulary* vocab
- *
- * def __init__(self, path): # <<<<<<<<<<<<<<
- * """
- * Load the language model.
- */
-
- /* function exit code */
- __pyx_r = 0;
- goto __pyx_L0;
- __pyx_L1_error:;
- __Pyx_XDECREF(__pyx_t_1);
- __Pyx_XDECREF(__pyx_t_2);
- __Pyx_XDECREF(__pyx_t_3);
- __Pyx_XDECREF(__pyx_t_4);
- __Pyx_XDECREF(__pyx_t_5);
- __Pyx_XDECREF(__pyx_t_12);
- __Pyx_XDECREF(__pyx_t_14);
- __Pyx_AddTraceback("kenlm.LanguageModel.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename);
- __pyx_r = -1;
- __pyx_L0:;
- __Pyx_XDECREF(__pyx_v_exception);
- __Pyx_XDECREF(__pyx_v_exception_message);
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-/* "kenlm.pyx":99
- * self.vocab = &self.model.BaseVocabulary()
- *
- * def __dealloc__(self): # <<<<<<<<<<<<<<
- * del self.model
- *
- */
-
-/* Python wrapper */
-static void __pyx_pw_5kenlm_13LanguageModel_3__dealloc__(PyObject *__pyx_v_self); /*proto*/
-static void __pyx_pw_5kenlm_13LanguageModel_3__dealloc__(PyObject *__pyx_v_self) {
- __Pyx_RefNannyDeclarations
- __Pyx_RefNannySetupContext("__dealloc__ (wrapper)", 0);
- __pyx_pf_5kenlm_13LanguageModel_2__dealloc__(((struct __pyx_obj_5kenlm_LanguageModel *)__pyx_v_self));
-
- /* function exit code */
- __Pyx_RefNannyFinishContext();
-}
-
-static void __pyx_pf_5kenlm_13LanguageModel_2__dealloc__(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self) {
- __Pyx_RefNannyDeclarations
- __Pyx_RefNannySetupContext("__dealloc__", 0);
-
- /* "kenlm.pyx":100
- *
- * def __dealloc__(self):
- * del self.model # <<<<<<<<<<<<<<
- *
- * property order:
- */
- delete __pyx_v_self->model;
-
- /* "kenlm.pyx":99
- * self.vocab = &self.model.BaseVocabulary()
- *
- * def __dealloc__(self): # <<<<<<<<<<<<<<
- * del self.model
- *
- */
-
- /* function exit code */
- __Pyx_RefNannyFinishContext();
-}
-
-/* "kenlm.pyx":103
- *
- * property order:
- * def __get__(self): # <<<<<<<<<<<<<<
- * return self.model.Order()
- *
- */
-
-/* Python wrapper */
-static PyObject *__pyx_pw_5kenlm_13LanguageModel_5order_1__get__(PyObject *__pyx_v_self); /*proto*/
-static PyObject *__pyx_pw_5kenlm_13LanguageModel_5order_1__get__(PyObject *__pyx_v_self) {
- PyObject *__pyx_r = 0;
- __Pyx_RefNannyDeclarations
- __Pyx_RefNannySetupContext("__get__ (wrapper)", 0);
- __pyx_r = __pyx_pf_5kenlm_13LanguageModel_5order___get__(((struct __pyx_obj_5kenlm_LanguageModel *)__pyx_v_self));
-
- /* function exit code */
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-static PyObject *__pyx_pf_5kenlm_13LanguageModel_5order___get__(struct __pyx_obj_5kenlm_LanguageModel *__pyx_v_self) {
- PyObject *__pyx_r = NULL;
- __Pyx_RefNannyDeclarations
- PyObject *__pyx_t_1 = NULL;
- int __pyx_lineno = 0;
- const char *__pyx_filename = NULL;
- int __pyx_clineno = 0;
- __Pyx_RefNannySetupContext("__get__", 0);
-
- /* "kenlm.pyx":104
- * property order:
- * def __get__(self):
- * return self.model.Order() # <<<<<<<<<<<<<<
- *
- * def score(self, sentence, bos = True, eos = True):
- */
- __Pyx_XDECREF(__pyx_r);
- __pyx_t_1 = __Pyx_PyInt_From_unsigned_int(__pyx_v_self->model->Order()); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 104; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_1);
- __pyx_r = __pyx_t_1;
- __pyx_t_1 = 0;
- goto __pyx_L0;
-
- /* "kenlm.pyx":103
- *
- * property order:
- * def __get__(self): # <<<<<<<<<<<<<<
- * return self.model.Order()
- *
- */
-
- /* function exit code */
- __pyx_L1_error:;
- __Pyx_XDECREF(__pyx_t_1);
- __Pyx_AddTraceback("kenlm.LanguageModel.order.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename);
- __pyx_r = NULL;
- __pyx_L0:;
- __Pyx_XGIVEREF(__pyx_r);
- __Pyx_RefNannyFinishContext();
- return __pyx_r;
-}
-
-/* "kenlm.pyx":106
- * return self.model.Order()
- *
- * def score(self, sentence, bos = True, eos = True): # <<<<<<<<<<<<<<
- * cdef list words = as_str(sentence).split()
- * cdef _kenlm.State state
- */
-
-/* Python wrapper */
-static PyObject *__pyx_pw_5kenlm_13LanguageModel_5score(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
-static PyObject *__pyx_pw_5kenlm_13LanguageModel_5score(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
- PyObject *__pyx_v_sentence = 0;
- PyObject *__pyx_v_bos = 0;
- PyObject *__pyx_v_eos
<TRUNCATED>