liblinear使用總結

liblinear是libsvm的線性核的改進版本，專門適用於百萬數據量的分類。正好適用於我這次數據挖掘的實驗。

 

liblinear用法和libsvm很相似，我是用的是.exe文件，利用python的subprocess向控制台發送命令即可完成本次試驗。

 

其中核心兩句即

train train.txt

predict test.txt train.txt.model output.txt

 

由於是線性核，沒有設置參數c、g

 

對於50W篇文章模型訓練僅需340秒，50W篇文章的預測僅需6秒

 

from subprocess import *
import time

time = time.time

start_time = time()
print("訓練")
cmd = "train train.txt"
Popen(cmd, shell = True, stdout = PIPE).communicate()
print("訓練結束",str(time() - start_time))


start_time = time()
print("預測")
cmd = "predict test.txt train.txt.model output.txt"
Popen(cmd, shell = True).communicate()
print("預測結束",str(time() - start_time))


#進行統計
#讀測試集真實label
start_time = time()
print("統計")
test_filename = "test.txt"
f = open(test_filename,"r",encoding = "utf-8")
real_class = []
for line in f:
    real_class.append(line[0])

#總樣本
total_sample = len(real_class)

#讀預測結果label
predict_filename = "output.txt"
f_predict = open(predict_filename,"r",encoding = "utf-8")
s = f_predict.read()
predict_class = s.split()

#對預測正確的文章進行計數
T = 0
for real, predict in zip(real_class,predict_class):
    if int(real) == int(predict):
        T += 1
accuracy  = T / total_sample * 100
print("正確率 為", str(accuracy) + "%")


# class_label = ["0","1","2","3","4","5","6","7","8","9"]
num_to_cate = {0:"it",1:"體育",2:"軍事",3:"金融",4:"健康",5:"汽車",6:"房產",7:"文化",8:"教育",9:"娛樂"}

class_label = ["it","體育","軍事","金融","健康","汽車","房產","文化","教育","娛樂"]

predict_precision = dict.fromkeys(class_label,1.0)
predict_true = dict.fromkeys(class_label,1.0)

predict_recall = dict.fromkeys(class_label,1.0)
predict_F = dict.fromkeys(class_label,0.0)
# print(str(predict_precision))
# print(str(predict_precision))
# print(str(predict_recall))
# print(str(predict_true))
mat = dict.fromkeys(class_label,{})
for k,v in mat.items():
    mat[k] = dict.fromkeys(class_label,0)

# print(str(mat))

for real, predict in zip(real_class,predict_class):
    real = int(real)
    predict = int(predict)
    # print(num_to_cate[real])
    # print(num_to_cate[predict])
    mat[num_to_cate[real]][num_to_cate[predict]] += 1
    predict_precision[num_to_cate[predict]] += 1
    predict_recall[num_to_cate[real]] += 1

    if int(real) == int(predict):
        predict_true[num_to_cate[predict]] += 1

# print(str(predict_precision))
# print(str(predict_recall))
# print(str(predict_true))

#輸出混淆矩陣
for k, v in mat.items():
    print(k + ":" + str(v))

#計算精確率和召回率
for x in range(len(class_label)):
    # x =  str(x)
    predict_precision[num_to_cate[x]] = predict_true[num_to_cate[x]] / predict_precision[num_to_cate[x]]
    predict_recall[num_to_cate[x]] = predict_true[num_to_cate[x]] / predict_recall[num_to_cate[x]]

# print(str(predict_precision))
# print(str(predict_recall))
# print(str(predict_true))

#計算F測度
for x in range(len(class_label)):
    # x = str(x)
    predict_F[num_to_cate[x]] = 2 * predict_recall[num_to_cate[x]] * predict_precision[num_to_cate[x]] / (predict_precision[num_to_cate[x]] + predict_recall[num_to_cate[x]])

print("統計結束",str(time() - start_time))
print("精確率為",str(predict_precision))
print("召回率為",str(predict_recall))
print("F測度為",str(predict_F))

print("保存結果")
final_result_filename = "./finalresult.txt"
f = open(final_result_filename,"w",encoding = "utf-8")
for k, v in mat.items():
    f.write(k + ":" + str(v) + "\n")

f.write("\n")
f.write("正確率為" + str(accuracy) + "%" + "\n\n")
f.write("精確率為" + str(predict_precision) + "\n\n")
f.write("召回率為" + str(predict_recall) + "\n\n")
f.write("F測度為" + str(predict_F) + "\n\n")
print("保存結果結束")


# cate_to_num = {"it":0,"體育":1,"軍事":2,"華人":3,"國內":4,"國際":5,"房產":6,"文娛":7,"社會":8,"財經":9}
# num_to_cate = {0:"it",1:"體育",2:"軍事",3:"華人",4:"國內",5:"國際",6:"房產",7:"文娛",8:"社會",9:"財經"}