文本相似度之LSI

1.VSM简介

     空间向量模型VSM，是将文本表示成数值表示的向量。在使用VSM做文本相似度计算时，其基本步骤是：

    1）将文本分词，提取特征词s:（t1,t2,t3,t4）

    2）将特征词用权重表示，从而将文本表示成数值向量s:(w1,w2,w3,w4)，权重表示的方式一般使用tfidf

    3）计算文本向量间的余弦值，判断文本间的相似度

缺点：空间向量模型以词袋为基础，没有考虑词与词间的关系，近义词等。

 

 2.LSI介绍

     潜在语义索引(Latent Semantic Indexing,以下简称LSI)，有的文章也叫Latent Semantic  Analysis（LSA）。是一种简单实用的主题模型。LSI是基于奇异值分解（SVD）的方法来得到文本的主题的。其推导过程如下图：(参考：http://www.cnblogs.com/pinard/p/6251584.html ,https://www.cnblogs.com/pinard/p/6805861.html)

     

 

使用LSI计算文本相似度的基本过程：

    1).和VSM一样，得到文本的tfidf值的向量表示 

    2).做SVD分解，得到Uk词和词义之间的相关性。Vk文本和主题的相关性。

    3).利用文本主题矩阵计算文本的相似度(通过余弦值)

 

3.应用

1.准备数据

    数据分为两部分（Idx2ID,questionList）

   idx2ID:保存数据库中的ID及ID在文件中的下标，例如（{0:100,1:101,2:102} 其中0,1,2为下标；100,102,103为数据库中的ID值）

  questionList：保存的是数据库中问题切词后的二维数组，例如[['我','跑路','了']，['你'，‘很’，‘不开心’]]

  其中questionList中的数组下标与idx2ID中的下标一 一对应，及数据库中的保存记录为：（id:100,question:‘我跑路了’）

 

 

import re
from tqdm import tqdm
import os
import pickle
import jieba
import pymysql
import pandas as pd 

'''
从数据库获取训练数据
'''

#清洗数据，去掉停用词并进行jieba分词
stwlist = pickle.load(open('stop_word.pkl','rb'))
def getTrainData(sentence):
    sentence=re.sub(r'[a-zA-Z0-9]+','', sentence)
    wordList = jieba.cut(sentence.strip(), cut_all=False)
    new_wordList = []
    for word in wordList:
        if word not in stwlist:
            new_wordList.append(word)
    return new_wordList

def get_ID_Question():
    aldb = pymysql.connect(host='xx.xx.xx.xx', port=3306, user='user1', passwd='123456', db="testdb", charset='utf8')
    IdList = []
    QuestionList = []
    with pymysql.cursors.SSCursor(aldb) as cursor:
        cursor.execute('select id,question from test')
        while True:
            rs = cursor.fetchone()
            if not rs:
                break
            id= rs[0]
            question = rs[1]
            IdList.append(id)
            QuestionList.append(getTrainData(question))
    #保存数据库中id及id在文件中对应的下标
    Idx2ID = {}
    for index, ID in enumerate(IdList):
        Idx2ID[index] = ID

    return Idx2ID,QuestionList

def save_ID_Question():
    Idx2ID_path = './cache/Idx2ID.pkl'
    QuesttionData_path='./cache/traindata.pkl'

    Idx2ID, QuestionList= get_ID_Question()

    #保存数据
    pickle.dump(Idx2ID, open(Idx2ID_path,'wb'))
    pickle.dump(QuestionList, open(QuesttionData_path, 'wb'))
    print('save data result!')

if __name__=='__main__':
    save_ID_Question()

 2.训练模型

#建立lsi模型，并将生成的LSI模型保存
def lsi_model(num_topics):
    lsi_model_dictionary = './cache/lsi_dictionary.pkl'
    lsi_model_lsi = './cache/model.lsi'
    lsi_model_index = './cache/lsimodel.index'
    cut_question_path = r'./cache/traindata.pkl'
    if os.path.exists(lsi_model_dictionary):
        dictionary = pickle.load(open(lsi_model_dictionary,'rb'))
        lsi = models.LsiModel.load(lsi_model_lsi)
        index = similarities.MatrixSimilarity.load(lsi_model_index)
    else:
        textdata1 = pickle.load(open(cut_question_path, 'rb'))
        #生成词典
        dictionary = corpora.Dictionary(textdata1)
        print('dictionary prepared!')
        corpus = [dictionary.doc2bow(text) for text in textdata1]
        print('corpus prepared!')
        #计算tfidf
        tfidf = models.TfidfModel(corpus)
        print('tfidf prepared!')
        corpus_tfidf = tfidf[corpus]
        #训练LSI
        lsi = models.LsiModel(corpus_tfidf, id2word=dictionary, num_topics = num_topics)
        print('lsi model prepared!')
        corpus_lsi = lsi[corpus_tfidf]
        #生成引用
        index = similarities.MatrixSimilarity(corpus_lsi)
        print('index prepared!')
        pickle.dump(dictionary, open(lsi_model_dictionary,'wb'))
        lsi.save(lsi_model_lsi)
        index.save(lsi_model_index)
    return dictionary, lsi, index

 3.测试

class test(object):
    def __init__(self):
        #加载模型数据
        self.dictionary = pickle.load(open('./cache/lsi_dictionary.pkl','rb'))
        self.lsi = models.LsiModel.load('./cache/model.lsi')
        self.index = similarities.MatrixSimilarity.load('./cache/lsimodel.index')
        self.stop_word = pickle.load(open('./stop_word.pkl','rb'))
        self.idx2ID = pickle.load(open('./cache/idx2ID.pkl','rb'))
    #清洗输入问题，并切词
    def deal_query(self,query):
        query = re.sub(r'[a-zA-Z0-9]+','',query)
        query_ques = jieba.cut(query.strip(), cut_all=False)
        new_wordList = []
        for word in query_ques:
            if word not in self.stop_word:
                new_wordList.append(word)
        return new_wordList

    #清洗匹配到的问题
    def clean_question(self, question):
        question = re.sub('律师', '', question)
        ques = pseg.cut(question)
        for word in ques:
            if word.flag == 'nr1' or word.flag == 'nr2' or word.flag == 'nr':
                keys = word.word
                question = re.sub(keys, '', question)
        return question

   #从数据库获取匹配到的问答对
    def get_answer(self,idx_set):
        question_set = []
        answer_set = []
        db = pymysql.connect(host='xx.xx.xx.x', port=3306, user='user1', passwd='123456', db="testdb", charset='utf8')
        for i in idx_set:
            ID = self.idx2ID[i]
            # print(ID)
            cursor = db.cursor()
            cursor.execute('select question,answer from test where id=%d;' %ID)
            db.commit()
            result = cursor.fetchone()
            if result:
                question = self.clean_question(result[0])
                question_set.append(question)
                answer = self.clean_answer(result[1])
                answer_set.append(answer)
        db.close()
        return question_set,answer_set
    #匹配
    def match(self,query):
        query_ques = self.deal_query(query)
        query_bow = self.dictionary.doc2bow(query_ques)
        query_lsi = self.lsi[query_bow]
        #计算余弦相似度，获取前3个最匹配的值
        sims = self.index[query_lsi]
        sort_sims = sorted(enumerate(sims), key=lambda item: -item[1])
        idx_set = [i[0] for i in sort_sims[:3]]
        sim_value = [i[1] for i in sort_sims[:3]]

        question_set,answer_set = self.get_answer(idx_set)

        return question_set,answer_set,sim_value

    def request(self,query):
        import json
        question_set,answer_set,sim_value = self.match(query)
        #相似度小于0.65的问题丢弃
        if sim_value[0] < 0.65:
            print("we have no match answer")
            return None
        i=0
        question_answer={}
        while i<len(question_set):
            q=question_set[i]
            a=answer_set[i]
            if len(q) > 2 and len(a) > 2:
                question_answer[q]=a
            if len(question_answer)>3:
                break
            i+=1
        # print_result2 = {k: v for k, v in question_answer.items() }
        print(len(question_answer))
        print(question_answer)
        return json.dumps(question_answer, ensure_ascii=False)

    #清洗答案数据
    def clean_answer(self,answer):
        # 将如果有多个回答，取长度最长的一个回答
        answer_sent = answer.strip().split('NEXT_ANS')
        if len(answer_sent) > 1:
            answer_len = [len(ans) for ans in answer_sent]
            inx = answer_len.index(max(answer_len))
            answer = answer_sent[inx]
        answer = re.sub('[，]+', '，', answer)
        answer = re.sub('[。]+', '。', answer)
        answer = re.sub('[.]+', '', answer)
        answer = re.sub('[,]+', '，', answer)

        from zhon.hanzi import punctuation
        # 去除 '\nNEXT_ANS'
        answer = answer.replace('NEXT_ANS', '')
        duels = [x + y for x in punctuation
                 for y in punctuation]
        pattern1 = re.compile('^[%s]' % punctuation)
        pattern2 = re.compile('[%s]$' % punctuation)
        pattern3 = re.compile('\[.*?\]')

        # 去除[]中的内容
        answer = pattern3.sub('', answer)

        # 去除空格
        answer = answer.split()
        answer = ''.join(answer)

        # 去除【】符号
        answer = answer.replace('【', '')
        answer = answer.replace('】', '')

        # 去除《》符号不全的句子中的《》
        if (answer.count('《') != answer.count('》')):
            answer = answer.replace('《', '')
            answer = answer.replace('》', '')

        # 去除‘去),张三你好’类型的错误
        delCStr = '《》（）&%￥#@！{}【】'
        if (answer[2] in delCStr):
            answer = answer[2:]

        # 去除\n \\n \u3000\u30005 \xa0 \r &quot
        answer = re.sub(r'\\u3\d{3,4}', '', answer)
        answer = re.sub(r'\\n', '', answer)
        answer = re.sub(r'\\xa0', '', answer)
        answer = re.sub(r'\\r', '', answer)
        answer = re.sub(r'&quot', '', answer)
        #去除电话
        answer = re.sub(r'电话：', '', answer)
        answer = re.sub(r'电话\d{11}', '', answer)  # 电话[phone]
        answer = re.sub(r'电话\d{3}\-\d{8}', '', answer)  # 电话[phone]
        answer = re.sub(r'\d{3}\-\d{8}', '', answer)  # 010-68812000
        answer = re.sub(r'0\d{10}', '', answer)  # 01065083250
        answer = re.sub(r'1[3458]\d{9}', '', answer)  # 手机号码
        answer = re.sub(r'[一二三四五六七八九零]{11,}', '', answer)  # 一三八二零二八八一二六

        # 去除QQ
        answer = re.sub(r'QQ：[1-9]\d{4,10}', '', answer)
        answer = re.sub(r'qq：[1-9]\d{4,10}', '', answer)
        answer = re.sub(r'QQ[1-9]\d{4,10}', '', answer)
        # 邮箱
        answer = re.sub(r'[0-9]*@.*.com', '', answer)
        # answer = re.sub(r'[1-9]\d{4,10}', '', answer)
        # 去除邮箱
        answer = re.sub(r'E-mail:', '', answer)
        answer = re.sub(r'email:', '', answer)
        answer = re.sub(r'电子邮箱', '', answer)
        answer = re.sub(r'邮箱', '', answer)
        answer = re.sub(r'[\w\.-]+@([\w]+\.)+[a-z]{2,3}', '', answer)
        # 去除网址
        answer = re.sub(r'个人网址：', '', answer)
        answer = re.sub(r'网址：', '', answer)
        answer = re.sub('[http|https|www].*[com|cn|html|0-9|a-z]', '', answer)
        # http://www51wfcom/zxzx/ZixunMainViewjsp?ID=bd84d23a2ec1fd73012ec2a422e40044

        answer = re.sub(r'[a-zA-Z]*', '', answer)

        # 特殊符号
        answer = re.sub(r'&.t', '', answer)
        answer = re.sub(r'&#215', '', answer)
        # 去除人名
        address = ''
        ans_list = pseg.cut(answer)
        for word in ans_list:
            # 去除不含国字的地名
            '''
            if (word.flag == 'ns' and not (word.word.find('国') > -1)):
                keys = word.word
                answer = re.sub(keys, '', answer)
                continue
            '''
            # 去除姓名：nr 人名，nr1 汉语姓氏，nr2 汉语名字
            if word.flag == 'nr1' or word.flag == 'nr2' or word.flag == 'nr':
                keys = word.word
                answer = re.sub(keys, '', answer)
        # 去除重复标点符号
        for d in duels:
            while d in answer:
                answer = answer.replace(d, d[0])

        # 去除开头标点符号
        if answer[0]!='《':
            answer = pattern1.sub('', answer)

        # 结尾加句号
        answer = pattern2.sub('。', answer)
        if pattern2.search(answer) is None:
            answer = answer + '。'
        else:
            answer = pattern2.sub('。', answer)

        return answer

if __name__=='__main__':
    # lsi_model(800)
    tst=test()
    while True:
        text=input('输入问题：')
        st=time.time()
        tst.request(text)
        end=time.time()
        print(end-st)