【原創】cython and python for kenlm

未經允許不可轉載

Kenlm相關知識

Kenlm下載地址
kenlm中文版本訓練語言模型
如何使用kenlm訓練出來的模型C++版本

關於Kenlm模塊的使用及C++源碼說明

加載Kenlm模塊命令

qy@IAT-QYVPN:~/Documents/kenlm/lm$ ../bin/query -n test.arpa

---

Kenlm模塊C++源碼說明

query的主入口文件:query_main.cc
query的執行函數文件:ngram_query.hh
注意:
默認執行的是query_main.cc文件96行的

Query<ProbingModel>(file, config, sentence_context, show_words);

而不是lm/wrappers/nplm.hh,這個封裝文件是需要NPLM模塊的,參考以下代碼,當時疏忽了在這個地方耽誤了一些時間

#ifdef WITH_NPLM
    } else if (lm::np::Model::Recognize(file)) {
      lm::np::Model model(file);
      if (show_words) {
        Query<lm::np::Model, lm::ngram::FullPrint>(model, sentence_context);
      } else {
        Query<lm::np::Model, lm::ngram::BasicPrint>(model, sentence_context);
      }
#endif

關於Model類的繼承關系

• 最基類virtual_interface.hh lm::base::Model
• 次基類facade.hh lm::base::ModelFacade : public Model
• 子類model.hh lm::ngram::GenericModel : public base::ModelFacade<GenericModel<Search, VocabularyT>, State, VocabularyT>

關於cython的簡單說明

cython官網
可以從官網下載最新版本,參考Documentation分類中的Cython Wiki和Cython FAQ了解一些知識。
cython-cpp-test-sample
Wrapping C++ Classes in Cython
cython wrapping of base and derived class
std::string arguments in cython
Cython and constructors of classes
Cython基礎--Cython入門

kenlm的python模塊封裝

接下來，讓我們進入正題，在kenlm的源碼中實際上已經提供了python的應用。在kenlm/python文件夾中，那么為什么還要再封裝python模塊呢，因為kenlm中所帶的python模塊僅僅實現了包含<s>和</s>這種情況下的計算分數的方法，而沒有提供不包含這種情況的計算分數的算法，這就是為什么要重新封裝python模塊的原因。

簡單介紹一下python模塊使用的必要步驟

• 安裝kenlm.so模塊到python的目錄下，默認直接運行kenlm目錄下的setup.py文件即可安裝成功sudo python setup.py install --record log。
• 安裝成功后，即可運行python example.py文件，查看運行結果。

如何擴展kenlm的python模塊

接下來，正式進入python擴展模塊的介紹。kenlm.pxd是cython針對所用到C++類及對象的聲明文件，kenlm.pyx是真正要編寫的cython功能代碼，也是未來python所要調用的類及方法。使用cython的編譯命令，可以把kenlm.pxd和kenlm.pyx編譯出kenlm.cpp文件。setup.py文件會用到編譯出來的kenlm.cpp文件。

• cython編譯命令cython --cplus kenlm.pyx

擴展后的kenlm.pxd文件

from libcpp.string cimport string

cdef extern from "lm/word_index.hh":
    ctypedef unsigned WordIndex

cdef extern from "lm/return.hh" namespace "lm":
    cdef struct FullScoreReturn:
        float prob
        unsigned char ngram_length

cdef extern from "lm/state.hh" namespace "lm::ngram":
    cdef struct State:
        pass

    ctypedef State const_State "const lm::ngram::State"

cdef extern from "lm/virtual_interface.hh" namespace "lm::base":
    cdef cppclass Vocabulary:
        WordIndex Index(char*)
        WordIndex BeginSentence() 
        WordIndex EndSentence()
        WordIndex NotFound()

    ctypedef Vocabulary const_Vocabulary "const lm::base::Vocabulary"


cdef extern from "lm/model.hh" namespace "lm::ngram":
    cdef cppclass Model:
        const_Vocabulary& GetVocabulary()
        const_State& NullContextState()
        void Model(char* file)
        FullScoreReturn FullScore(const_State& in_state, WordIndex new_word, const_State& out_state)

        void BeginSentenceWrite(void *)
        void NullContextWrite(void *)
        unsigned int Order()
        const_Vocabulary& BaseVocabulary()
        float BaseScore(void *in_state, WordIndex new_word, void *out_state)
        FullScoreReturn BaseFullScore(void *in_state, WordIndex new_word, void *out_state)
        void * NullContextMemory()

擴展后的kenlm.pyx文件

import os

cdef bytes as_str(data):
    if isinstance(data, bytes):
        return data
    elif isinstance(data, unicode):
        return data.encode('utf8')
    raise TypeError('Cannot convert %s to string' % type(data))

cdef int as_in(int &Num):
    (&Num)[0] = 1

cdef class LanguageModel:
    cdef Model* model
    cdef public bytes path
    cdef const_Vocabulary* vocab

    def __init__(self, path):
        self.path = os.path.abspath(as_str(path))
        try:
            self.model = new Model(self.path)
        except RuntimeError as exception:
            exception_message = str(exception).replace('\n', ' ')
            raise IOError('Cannot read model \'{}\' ({})'.format(path, exception_message))\
                    from exception
        self.vocab = &self.model.GetVocabulary()

    def __dealloc__(self):
        del self.model

    property order:
        def __get__(self):
            return self.model.Order()
    
    def score(self, sentence):
        cdef list words = as_str(sentence).split()
        cdef State state
        self.model.BeginSentenceWrite(&state)
        cdef State out_state
        cdef float total = 0
        for word in words:
            total += self.model.BaseScore(&state, self.vocab.Index(word), &out_state)
            state = out_state
        total += self.model.BaseScore(&state, self.vocab.EndSentence(), &out_state)
        return total

    def full_scores(self, sentence):
        cdef list words = as_str(sentence).split()
        cdef State state
        self.model.BeginSentenceWrite(&state)
        cdef State out_state
        cdef FullScoreReturn ret
        cdef float total = 0
        for word in words:
            ret = self.model.BaseFullScore(&state,
                self.vocab.Index(word), &out_state)
            yield (ret.prob, ret.ngram_length)
            state = out_state
        ret = self.model.BaseFullScore(&state,
            self.vocab.EndSentence(), &out_state)
        yield (ret.prob, ret.ngram_length)
    
    def full_scores_n(self, sentence):
        cdef list words = as_str(sentence).split()
        cdef State state
        state = self.model.NullContextState()
        cdef State out_state
        cdef FullScoreReturn ret
        cdef int ovv = 0
        for word in words:
            ret = self.model.FullScore(state,
                self.vocab.Index(word), out_state)
            yield (ret.prob, ret.ngram_length)
            state = out_state

    """""""""""
    """count scores when not included <s> and </s>"""
    """""""""""
    def score_n(self, sentence):
        cdef list words = as_str(sentence).split()
        cdef State state
        state = self.model.NullContextState()
        cdef State out_state
        cdef float total = 0
        for word in words:
            ret = self.model.FullScore(state,
                self.vocab.Index(word), out_state)
            total += ret.prob
            """print(total)"""
            state = out_state
        return total


    def __contains__(self, word):
        cdef bytes w = as_str(word)
        return (self.vocab.Index(w) != 0)

    def __repr__(self):
        return '<LanguageModel from {0}>'.format(os.path.basename(self.path))

    def __reduce__(self):
        return (LanguageModel, (self.path,))