自然語言預處理

#英文預處理

#按空格進行分詞，同時針對推文一些特性，去除@用戶名，保留表情等一些特殊符號
tokenizer = TweetTokenizer()
for counter,rev in enumerate(reviews):
    # 去除HTML網頁格式
    temp = BeautifulSoup(rev)
    text = temp.get_text()
    # 去除空格
    text = re.sub(' +',' ',text)
    test = re.sub(r'[()\[\]{}.,;:!?\<=>？@_^#$%"&*-],' ',text)
    # strip leading and trailing white space
    text = text.strip()
    # tokenize 
    tokens = tokenizer.tokenize(text)
    cleaned_reviews.append(tokens)
    if counter % round(len(reviews)/10) == 0:
        print(counter, '/', len(reviews), 'reviews cleaned')
# get list of tokens from all reviews
# 兩個list變成一個list
all_tokens = [token for sublist in cleaned_reviews for token in sublist]
# 根據詞頻做index， 把單詞轉成index
counts = dict(Counter(all_tokens))
sorted_counts = sorted(counts.items(), key=operator.itemgetter(1), reverse=True)
# assign to each word an index based on its frequency in the corpus
# the most frequent word will get index equal to 1
word_to_index = dict([(tuple[0],idx+1) for idx, tuple in enumerate(sorted_counts)])
with open(path_to_IMDB + 'word_to_index_new.json', 'w') as my_file:
     json.dump(word_to_index, my_file, sort_keys=True, indent=4)

詞共現矩陣的構建

https://github.com/urgedata/pythondata/blob/master/Text%20Analytics/ericbrown.ipynb 

#中文預處理

#jieba分詞和去停用詞
#jieba 分詞可以將我們的自定義詞典導入，格式 “詞” “詞性” “詞頻”
jieba.load_userdict('data/userdict.txt')
#定義一個keyword類
class keyword(object):
    def Chinese_Stopwords(self):          #導入停用詞庫
        stopword=[]
        cfp=open('data/stopWord.txt','r+','utf-8')   #停用詞的txt文件
        for line in cfp:
            for word in line.split():
                stopword.append(word)
        cfp.close()
        return stopword
def Word_cut_list(self,word_str):
        #利用正則表達式去掉一些一些標點符號之類的符號。
        word_str = re.sub(r'\s+', ' ', word_str)  # trans 多空格 to空格
        word_str = re.sub(r'\n+', ' ', word_str)  # trans 換行 to空格
        word_str = re.sub(r'\t+', ' ', word_str)  # trans Tab to空格
        word_str = re.sub("[\s+\.\!\/_,$%^*(+\"\']+|[+——；！，”。《》，。：“？、~@#￥%……&*（）1234567①②③④)]+".decode("utf8"), "".decode("utf8"), word_str)
        wordlist = list(jieba.cut(word_str))#jieba分詞
        wordlist_N = []
        chinese_stopwords=self.Chinese_Stopwords()
        for word in wordlist:
            if word not in chinese_stopwords:#詞語的清洗：去停用詞
                if word != '\r\n'  and word!=' ' and word != '\u3000'.decode('unicode_escape') \
                        and word!='\xa0'.decode('unicode_escape'):#詞語的清洗：去全角空格
                    wordlist_N.append(word)
        return wordlist_N

#名詞提取
def Word_pseg(self,word_str):  # 名詞提取函數
        words = pseg.cut(word_str)
        word_list = []
        for wds in words:
            # 篩選自定義詞典中的詞，和各類名詞，自定義詞庫的詞在沒設置詞性的情況下默認為x詞性，即詞的flag詞性為x
            if wds.flag == 'x' and wds.word != ' ' and wds.word != 'ns' \
                    or re.match(r'^n', wds.flag) != None \
                            and re.match(r'^nr', wds.flag) == None:
                word_list.append(wds.word)
        return word_list

import tensorflow.contrib.keras as kr

def read_file(filename):
    """讀取文件數據"""
    contents, labels = [], []
    with open_file(filename) as f:
        for line in f:
            try:
                label, content = line.strip().split('\t')
                if content:
                    contents.append(list(content))#通過list把一句話分成一個個字
                    labels.append(native_content(label))
            except:
                pass
    return contents, labels


def build_vocab(train_dir, vocab_dir, vocab_size=5000):
    """根據訓練集構建詞匯表，存儲"""
    data_train, _ = read_file(train_dir)
    all_data = []
    for content in data_train:
        all_data.extend(content)

    counter = Counter(all_data)
    count_pairs = counter.most_common(vocab_size - 1) #輸出幾個出現次數最多的元素
    words, _ = list(zip(*count_pairs)) #通過zip只取出其中的單詞
    # 添加一個 <PAD> 來將所有文本pad為同一長度
    words = ['<PAD>'] + list(words)
    open_file(vocab_dir, mode='w').write('\n'.join(words) + '\n')

def read_vocab(vocab_dir):
    """讀取詞匯表"""
    # words = open_file(vocab_dir).read().strip().split('\n')
    with open_file(vocab_dir) as fp:
        # 如果是py2 則每個值都轉化為unicode
        words = [native_content(_.strip()) for _ in fp.readlines()]
    word_to_id = dict(zip(words, range(len(words)))) 
    return words, word_to_id

def process_file(filename, word_to_id, cat_to_id, max_length=600):
    """將文件轉換為id表示"""
    contents, labels = read_file(filename)
    data_id, label_id = [], []
    for i in range(len(contents)):
        data_id.append([word_to_id[x] for x in contents[i] if x in word_to_id])
        label_id.append(cat_to_id[labels[i]])
    # 使用keras提供的pad_sequences來將文本pad為固定長度
    x_pad = kr.preprocessing.sequence.pad_sequences(data_id, max_length)
    y_pad = kr.utils.to_categorical(label_id, num_classes=len(cat_to_id))  # 將標簽轉換為one-hot表示
    return x_pad, y_pad

#建立詞表
text = open(path,encoding='utf-8').read().lower()
chars = set(text)
print ('total chars:', len(chars))
char_indices = dict((c, i) for i, c in enumerate(chars))
indices_char = dict((i, c) for i, c in enumerate(chars))
#kreas下運行LSTM的Input生成，在建立詞表的基礎上，數據向量化
print('Vectorization...')
X = np.zeros((len(sentences), maxlen, len(chars)), dtype=np.bool)
print(X)
y = np.zeros((len(sentences), len(chars)), dtype=np.bool)
for i, sentence in enumerate(sentences):
    for t, char in enumerate(sentence):
        X[i, t, char_indices[char]] = 1
    y[i, char_indices[next_chars[i]]] = 1

# 過濾詞長，過濾停用詞，只保留中文
def is_fine_word(word, min_length=2):
    rule = re.compile(r"^[\u4e00-\u9fa5]+$")
    if len(word) >= min_length and word not in STOP_WORDS and re.search(rule, word):
        return True
    else:
        return False

 

#逐字切分的處理方式，同時去掉一些常見的虛詞，如“之”、“乎”、“者”、“也”。
def singCut(text):
     tex = [i.strip('\n').strip('\r').strip('。').strip('，|）|：|{|}|“|” |（|\n') for i in text]
     return list(filter(None, tex)) #去掉空字符
text = '雲橫秦嶺家何在，雪擁藍關馬不前'

#虛詞通用詞庫
stopwords = '而|何|乎|乃|且|其|若|所|為'
#去掉標點
poem = [[i.strip(') |: |?|{|}| “|” （| \n\n\r|。') for i in tex if i not in stopwords]for tex in text]
poem = list(filter(None, poem ))

　　

預處理（去特殊符號、去停用詞、分詞）　　

把詞轉成index（word to index）, 把原文都變成數值

去掉topN詞頻的以及小於TOPM詞頻的

對每篇進行 truncation and padding

word2vec訓練 得到 w2v_model[word] 的embedding，加入CNN作為初始值（kreas里面訓練需要把每個詞轉成embedding這種）

訓練CNN模型

https://github.com/Tixierae/deep_learning_NLP

 

構建詞匯表

categories轉成id, 讀取詞匯表，構建word_to_id字典（字符級別）

讀入訓練數據，預處理，將文本pad到固定長度

批次訓練CNN（tensorflow內部會自動初始化embedding）

預測

 https://github.com/gaussic/text-classification-cnn-rnn

 

 

 引用鏈接：

https://www.jianshu.com/p/aea87adee163